You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ignite.apache.org by "Cyrill (via GitHub)" <gi...@apache.org> on 2025/12/16 08:18:06 UTC
[PR] IGNITE-27156 Change timeout semantics [ignite-3]
Cyrill opened a new pull request, #7242:
URL: https://github.com/apache/ignite-3/pull/7242
https://issues.apache.org/jira/browse/IGNITE-27156
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2736937675
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/RaftGroupServiceTest.java:
##########
@@ -680,6 +680,51 @@ public void testRetryOnErrorWithTimeout(RaftError error) {
assertThat(response, willThrow(TimeoutException.class, "Send with retry timed out"));
}
+ /**
+ * Tests that UNKNOWN/EINTERNAL/ENOENT errors retry on the same peer (the leader) for ActionRequests.
+ * This is because these errors are transient and the leader is likely to recover.
+ */
+ @ParameterizedTest
+ @EnumSource(names = {"UNKNOWN", "EINTERNAL", "ENOENT"})
+ public void testRetryOnTransientErrorRetriesOnSamePeer(RaftError error) {
+ Peer leaderPeer = NODES.get(0);
+
+ // First call returns error, second call succeeds - both to the same leader.
+ when(messagingService.invoke(
+ argThat((InternalClusterNode node) -> node != null && node.name().equals(leaderPeer.consistentId())),
+ any(ReadActionRequest.class),
+ anyLong())
+ )
+ .thenReturn(completedFuture(FACTORY.errorResponse()
+ .errorCode(error.getNumber())
+ .build()))
+ .thenReturn(completedFuture(FACTORY.actionResponse().result(null).build()));
+
+ RaftGroupService service = startRaftGroupServiceWithRefreshLeader(NODES);
+
+ assertThat(service.leader(), is(leaderPeer));
+
+ CompletableFuture<Object> response = service.run(mock(ReadCommand.class));
+
+ assertThat(response, willBe(nullValue()));
+
+ // Verify that only the leader was called (twice: once with error, once with success).
+ verify(messagingService, atLeastOnce()).invoke(
+ argThat((InternalClusterNode target) -> target != null && target.name().equals(leaderPeer.consistentId())),
+ any(ReadActionRequest.class),
+ anyLong()
+ );
+
+ // Verify that other peers were NOT called.
+ for (Peer otherPeer : NODES.subList(1, NODES.size())) {
+ verify(messagingService, org.mockito.Mockito.never()).invoke(
Review Comment:
> org.mockito.Mockito.
Let's do static import there.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
Review Comment:
Let's `{@code ... }` it in the javadoc here and below.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
Review Comment:
Here and below I'd to suggest introduce it as `int expectedTerm` to check in assertion the same as argument for tested methods.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
Review Comment:
The same, this assertion may be done in the common test before leader election.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
Review Comment:
We have to assert that `waiter1` future isn't done before `stop` is called.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
Review Comment:
Should we afraid of negative values passed through `onLeaderElected`? And check it with `IllegalArgumentException` inside?
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
Review Comment:
Not sure why this test. The code there is sequential, what may be go wrong there?
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
Review Comment:
... or even better with `{@link .. }`
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
Review Comment:
Should we move assertions from `testAwaitLeaderWhenLeaderAvailable` above and then merge it to the common one test? We still test the same method and assertions are about side effects after the same call: above one about await future and this one about how state is affected.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
Review Comment:
A bit strange test. Let's assert at least that state before the call `LeaderAvailabilityState.State expectedNotChangedState = state.currentState()` is the same as after the call.
And javadoc with `{@code .. }` or even better with `{@link .. }`.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
Review Comment:
Should we compare the value from `maxTerm` atomic? It's looking missed. Or unnecessary one.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
Review Comment:
Should we somehow test internal assumption that given term should be equal the current elected one? Not more or less, but equal.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
Review Comment:
Test name looks a bit irrelevant: `testStateStopCompletesAllWaitersExceptionally`, without `Waiters` it looks like `stop` call itself completes exceptionally. And I believe it may be merged with `AllWaiters` test below.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
Review Comment:
Should we check that the state is remaining the same `LEADER_AVAILABLE`?
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // New waiter should immediately fail with the same exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone(), "New waiter should be immediately completed");
+ assertTrue(waiter2.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when no waiters are pending still marks state as stopped. */
+ @Test
+ void testStopWhenNoWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // Stop without any waiters - should not throw
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // Subsequent awaitLeader should fail
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when leader is available fails subsequent awaitLeader. */
+ @Test
+ void testStopWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // awaitLeader should fail after stop, even though leader was available
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Multiple stop calls are idempotent. */
+ @Test
+ void testMultipleStopCalls() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException exception1 = new RuntimeException("Shutdown 1");
+ RuntimeException exception2 = new RuntimeException("Shutdown 2");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(exception1);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Second stop call should be ignored
+ state.stop(exception2);
+
+ // Subsequent awaitLeader should fail with the FIRST exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(exception1, thrown.getCause(), "Should use exception from first stop call");
+ }
+
+ /** onLeaderElected is ignored after stop. */
+ @Test
+ void testOnLeaderElectedIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Leader election after stop should be ignored
+ state.onLeaderElected(leader, 5);
+
+ // State should still be stopped.
+ assertTrue(state.stopped());
+
+ // New waiter should still fail
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter2.isCompletedExceptionally());
+ }
+
+ /** onGroupUnavailable is ignored after stop. */
+ @Test
+ void testOnGroupUnavailableIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ // Group unavailable after stop should be ignored
+ state.onGroupUnavailable(1);
+
+ assertTrue(state.stopped());
+ }
+
+ /** Stop preserves exception type. */
+ @Test
+ void testStopPreservesExceptionType() {
Review Comment:
Won't we check it in `testMultipleStopCalls` already?
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java:
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.jetbrains.annotations.VisibleForTesting;
+
+/**
+ * State machine for tracking leader availability.
+ *
+ * <p>Manages transitions between two states:
+ * <ul>
+ * <li>{@link State#WAITING_FOR_LEADER} - No leader is currently known, operations must wait</li>
+ * <li>{@link State#LEADER_AVAILABLE} - A leader is known, operations can proceed</li>
+ * </ul>
+ *
+ * <p>State transitions:
+ * <pre>
+ * WAITING_FOR_LEADER --[onLeaderElected]--> LEADER_AVAILABLE
+ * LEADER_AVAILABLE --[onGroupUnavailable]--> WAITING_FOR_LEADER
+ * Any state --[stop]--> stopped (terminal state)
+ * </pre>
+ *
+ * <p>Thread-safe: all public methods are synchronized or use proper synchronization.
+ */
+@VisibleForTesting
+class LeaderAvailabilityState {
+ /** The logger. */
+ private static final IgniteLogger LOG = Loggers.forClass(LeaderAvailabilityState.class);
+
+ /** Possible states of leader availability. */
+ enum State {
+ /** No leader is known, waiting for leader election. */
+ WAITING_FOR_LEADER,
+ /** A leader is available and operations can proceed. */
+ LEADER_AVAILABLE
+ }
+
+ /** Current state. Guarded by {@code this}. */
+ private State currentState = State.WAITING_FOR_LEADER;
+
+ /** Current leader term. Initialized to -1 to accept term 0 as the first valid term. Guarded by {@code this}. */
+ private long currentTerm = -1;
+
+ /** Future that waiters block on. Guarded by {@code this}. */
+ private CompletableFuture<Long> waiters = new CompletableFuture<>();
+
+ /** Whether the state machine has been stopped. Guarded by {@code this}. */
+ private boolean stopped = false;
+
+ /** Exception used to fail futures after destruction. Guarded by {@code this}. */
+ private Throwable stopException;
+
+ /**
+ * Returns a future that completes when a leader becomes available.
+ *
+ * <p>If a leader is already available, returns an already-completed future with the current term.
+ * Otherwise, returns a future that will complete when {@link #onLeaderElected} is called.
+ * If the state machine has been destroyed, returns a failed future.
+ *
+ * @return Future that completes with the leader term when a leader is available.
+ */
+ synchronized CompletableFuture<Long> awaitLeader() {
+ if (stopped) {
+ return CompletableFuture.failedFuture(stopException);
+ }
+ if (currentState == State.LEADER_AVAILABLE) {
+ return CompletableFuture.completedFuture(currentTerm);
+ }
+ return waiters;
+ }
+
+ /**
+ * Handles leader election notification.
+ *
+ * <p>Transitions from {@link State#WAITING_FOR_LEADER} to {@link State#LEADER_AVAILABLE}
+ * if the new term is greater than the current term. Stale notifications (with term <= current)
+ * are ignored. Has no effect if the state machine has been stopped.
+ *
+ * @param leader The newly elected leader node.
+ * @param term The term of the new leader.
+ */
+ void onLeaderElected(InternalClusterNode leader, long term) {
+ CompletableFuture<Long> futureToComplete = null;
+
+ synchronized (this) {
Review Comment:
Let's make synchronization on internal private mutex, not on `this` reference.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
Review Comment:
I think this test is covered already by `testMultipleWaitersCompleted`.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // New waiter should immediately fail with the same exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone(), "New waiter should be immediately completed");
+ assertTrue(waiter2.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when no waiters are pending still marks state as stopped. */
+ @Test
+ void testStopWhenNoWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // Stop without any waiters - should not throw
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // Subsequent awaitLeader should fail
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when leader is available fails subsequent awaitLeader. */
+ @Test
+ void testStopWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // awaitLeader should fail after stop, even though leader was available
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Multiple stop calls are idempotent. */
+ @Test
+ void testMultipleStopCalls() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException exception1 = new RuntimeException("Shutdown 1");
+ RuntimeException exception2 = new RuntimeException("Shutdown 2");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(exception1);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Second stop call should be ignored
+ state.stop(exception2);
+
+ // Subsequent awaitLeader should fail with the FIRST exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(exception1, thrown.getCause(), "Should use exception from first stop call");
+ }
+
+ /** onLeaderElected is ignored after stop. */
+ @Test
+ void testOnLeaderElectedIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Leader election after stop should be ignored
+ state.onLeaderElected(leader, 5);
+
+ // State should still be stopped.
+ assertTrue(state.stopped());
+
+ // New waiter should still fail
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter2.isCompletedExceptionally());
+ }
+
+ /** onGroupUnavailable is ignored after stop. */
+ @Test
+ void testOnGroupUnavailableIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ // Group unavailable after stop should be ignored
+ state.onGroupUnavailable(1);
+
+ assertTrue(state.stopped());
+ }
+
+ /** Stop preserves exception type. */
+ @Test
+ void testStopPreservesExceptionType() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ IllegalStateException testException = new IllegalStateException("Component stopping");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ state.stop(testException);
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertThat(thrown.getCause(), instanceOf(IllegalStateException.class));
+ assertEquals("Component stopping", thrown.getCause().getMessage());
+ }
+
+ /** stopped() returns false before stop and true after. */
+ @Test
+ void testStopped() {
Review Comment:
I'd to merge this assertions into one of first stop tests.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2743024906
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
Review Comment:
Resolved. (not sure why it was moved to top but still)
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
Review Comment:
Resolved, new test `testNegativeTermRejected` is confirmed.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
Review Comment:
Resolved. The atomic has been removed.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
Review Comment:
Resolved, the test was removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741129438
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741222081
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
Review Comment:
merged this test
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741261978
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737293107
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
Review Comment:
Resolved for now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2740641698
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
Review Comment:
fixed some of them
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741135539
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741282075
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737313360
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Recoverable error during the request occurred (will be retried) [request={}, peer={}, newPeer={}, traceId={}].",
+ includeSensitive() ? retryContext.request() : retryContext.request().toStringForLightLogging(),
+ retryContext.targetPeer(),
+ nextPeer,
+ retryContext.errorTraceId()
+ );
+ }
+ }
+
+ /**
+ * Handles throwable in single attempt mode.
+ */
+ private void handleThrowableSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ Throwable err,
+ RetryContext retryContext
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ sendWithRetrySingleAttempt(fut, retryContext.nextAttemptForUnavailablePeer(nextPeer, shortReasonMessage));
+ }
+
+ /**
+ * Handles error response in single attempt mode.
+ *
+ * <p>In single-attempt mode, each peer is tried at most once. All errors mark the peer
+ * as tried and move to the next peer. When all peers have been tried, the request fails
+ * with {@link ReplicationGroupUnavailableException}.
+ */
+ private void handleErrorResponseSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ ErrorResponse resp,
+ RetryContext retryContext
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ // Single-attempt mode: ALWAYS mark current peer as unavailable, regardless of trackCurrentAs.
+ // This prevents retrying the same peer and matches original behavior where:
+ // - transient errors: would select new peer, mark current unavailable
+ // - peer unavailable: marks current unavailable
+ // - no leader: marks current unavailable (no NO_LEADER distinction needed)
+ // - leader redirect: marks current unavailable (to prevent redirect loops)
+ sendWithRetrySingleAttempt(fut, context.nextAttemptForUnavailablePeer(nextPeer, reason));
+ }
+
+ @Override
+ public void onAllPeersExhausted() {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+ }
+ };
+
+ handleErrorResponseCommon(fut, resp, retryContext, strategy, false);
+ }
+
+ /**
+ * Sends a request with retry and waits for leader on leader absence.
+ */
+ private void sendWithRetryWaitingForLeader(
+ CompletableFuture<ActionResponse> fut,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long requestStartTime = Utils.monotonicMs();
+ long stopTime = retryContext.stopTime();
+
+ if (requestStartTime >= stopTime) {
+ // Retry timeout expired - fail with timeout exception.
+ // For non-leader-related retriable errors, we don't wait for leader, just fail.
+ fut.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ ThrottlingContextHolder peerThrottlingContextHolder = throttlingContextHolder.peerContextHolder(
+ retryContext.targetPeer().consistentId()
+ );
+
+ peerThrottlingContextHolder.beforeRequest();
+ retryContext.onNewAttempt();
+
+ // Bound response timeout by remaining time until retry stop time.
+ long responseTimeout = Math.min(peerThrottlingContextHolder.peerRequestTimeoutMillis(), stopTime - requestStartTime);
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ // Enforce timeout even if messaging service doesn't (e.g., in tests with mocks).
+ .orTimeout(responseTimeout > 0 ? responseTimeout : Long.MAX_VALUE, TimeUnit.MILLISECONDS)
+ .whenComplete((resp, err) -> {
+ peerThrottlingContextHolder.afterRequest(requestStartTime, retriableError(err, resp));
+
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableWithLeaderWait(fut, err, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseWithLeaderWait(fut, (ErrorResponse) resp, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((ActionResponse) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Handles throwable in leader-wait mode.
+ */
+ private void handleThrowableWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ Throwable err,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ scheduleRetryWithLeaderWait(fut, retryContext.nextAttempt(nextPeer, shortReasonMessage), cmd, deadline, termWhenStarted);
+ }
+
+ /**
+ * Handles error response in leader-wait mode.
+ *
+ * <p>In leader-wait mode:
+ * <ul>
+ * <li>Transient errors (EBUSY/EAGAIN) retry on the same peer after delay</li>
+ * <li>"No leader" errors try each peer once, then wait for leader notification</li>
+ * <li>Peer unavailability errors cycle through peers until timeout</li>
+ * </ul>
+ */
+ private void handleErrorResponseWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ ErrorResponse resp,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
Review Comment:
Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2740546370
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2740790316
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
we have the following code:
```
long responseTimeout = Math.min(retryContext.responseTimeoutMillis(), stopTime - requestStartTime);
```
We get the minimal value out of the single request response timeout and total available time for the whole series.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741294924
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2743026038
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // New waiter should immediately fail with the same exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone(), "New waiter should be immediately completed");
+ assertTrue(waiter2.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when no waiters are pending still marks state as stopped. */
+ @Test
+ void testStopWhenNoWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // Stop without any waiters - should not throw
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // Subsequent awaitLeader should fail
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when leader is available fails subsequent awaitLeader. */
+ @Test
+ void testStopWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // awaitLeader should fail after stop, even though leader was available
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Multiple stop calls are idempotent. */
+ @Test
+ void testMultipleStopCalls() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException exception1 = new RuntimeException("Shutdown 1");
+ RuntimeException exception2 = new RuntimeException("Shutdown 2");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(exception1);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Second stop call should be ignored
+ state.stop(exception2);
+
+ // Subsequent awaitLeader should fail with the FIRST exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(exception1, thrown.getCause(), "Should use exception from first stop call");
+ }
+
+ /** onLeaderElected is ignored after stop. */
+ @Test
+ void testOnLeaderElectedIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Leader election after stop should be ignored
+ state.onLeaderElected(leader, 5);
+
+ // State should still be stopped.
+ assertTrue(state.stopped());
+
+ // New waiter should still fail
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter2.isCompletedExceptionally());
+ }
+
+ /** onGroupUnavailable is ignored after stop. */
+ @Test
+ void testOnGroupUnavailableIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ // Group unavailable after stop should be ignored
+ state.onGroupUnavailable(1);
+
+ assertTrue(state.stopped());
+ }
+
+ /** Stop preserves exception type. */
+ @Test
+ void testStopPreservesExceptionType() {
Review Comment:
Resolved, the test was removed.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // New waiter should immediately fail with the same exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone(), "New waiter should be immediately completed");
+ assertTrue(waiter2.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when no waiters are pending still marks state as stopped. */
+ @Test
+ void testStopWhenNoWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // Stop without any waiters - should not throw
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // Subsequent awaitLeader should fail
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when leader is available fails subsequent awaitLeader. */
+ @Test
+ void testStopWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // awaitLeader should fail after stop, even though leader was available
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Multiple stop calls are idempotent. */
+ @Test
+ void testMultipleStopCalls() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException exception1 = new RuntimeException("Shutdown 1");
+ RuntimeException exception2 = new RuntimeException("Shutdown 2");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(exception1);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Second stop call should be ignored
+ state.stop(exception2);
+
+ // Subsequent awaitLeader should fail with the FIRST exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(exception1, thrown.getCause(), "Should use exception from first stop call");
+ }
+
+ /** onLeaderElected is ignored after stop. */
+ @Test
+ void testOnLeaderElectedIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Leader election after stop should be ignored
+ state.onLeaderElected(leader, 5);
+
+ // State should still be stopped.
+ assertTrue(state.stopped());
+
+ // New waiter should still fail
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter2.isCompletedExceptionally());
+ }
+
+ /** onGroupUnavailable is ignored after stop. */
+ @Test
+ void testOnGroupUnavailableIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ // Group unavailable after stop should be ignored
+ state.onGroupUnavailable(1);
+
+ assertTrue(state.stopped());
+ }
+
+ /** Stop preserves exception type. */
+ @Test
+ void testStopPreservesExceptionType() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ IllegalStateException testException = new IllegalStateException("Component stopping");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ state.stop(testException);
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertThat(thrown.getCause(), instanceOf(IllegalStateException.class));
+ assertEquals("Component stopping", thrown.getCause().getMessage());
+ }
+
+ /** stopped() returns false before stop and true after. */
+ @Test
+ void testStopped() {
Review Comment:
Resolved.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
Review Comment:
Resolved.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
Review Comment:
Resolved, covered by `testOnGroupUnavailableIgnoredIfTermChanged`.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java:
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.jetbrains.annotations.VisibleForTesting;
+
+/**
+ * State machine for tracking leader availability.
+ *
+ * <p>Manages transitions between two states:
+ * <ul>
+ * <li>{@link State#WAITING_FOR_LEADER} - No leader is currently known, operations must wait</li>
+ * <li>{@link State#LEADER_AVAILABLE} - A leader is known, operations can proceed</li>
+ * </ul>
+ *
+ * <p>State transitions:
+ * <pre>
+ * WAITING_FOR_LEADER --[onLeaderElected]--> LEADER_AVAILABLE
+ * LEADER_AVAILABLE --[onGroupUnavailable]--> WAITING_FOR_LEADER
+ * Any state --[stop]--> stopped (terminal state)
+ * </pre>
+ *
+ * <p>Thread-safe: all public methods are synchronized or use proper synchronization.
+ */
+@VisibleForTesting
+class LeaderAvailabilityState {
+ /** The logger. */
+ private static final IgniteLogger LOG = Loggers.forClass(LeaderAvailabilityState.class);
+
+ /** Possible states of leader availability. */
+ enum State {
+ /** No leader is known, waiting for leader election. */
+ WAITING_FOR_LEADER,
+ /** A leader is available and operations can proceed. */
+ LEADER_AVAILABLE
+ }
+
+ /** Current state. Guarded by {@code this}. */
+ private State currentState = State.WAITING_FOR_LEADER;
+
+ /** Current leader term. Initialized to -1 to accept term 0 as the first valid term. Guarded by {@code this}. */
+ private long currentTerm = -1;
+
+ /** Future that waiters block on. Guarded by {@code this}. */
+ private CompletableFuture<Long> waiters = new CompletableFuture<>();
+
+ /** Whether the state machine has been stopped. Guarded by {@code this}. */
+ private boolean stopped = false;
+
+ /** Exception used to fail futures after destruction. Guarded by {@code this}. */
+ private Throwable stopException;
+
+ /**
+ * Returns a future that completes when a leader becomes available.
+ *
+ * <p>If a leader is already available, returns an already-completed future with the current term.
+ * Otherwise, returns a future that will complete when {@link #onLeaderElected} is called.
+ * If the state machine has been destroyed, returns a failed future.
+ *
+ * @return Future that completes with the leader term when a leader is available.
+ */
+ synchronized CompletableFuture<Long> awaitLeader() {
+ if (stopped) {
+ return CompletableFuture.failedFuture(stopException);
+ }
+ if (currentState == State.LEADER_AVAILABLE) {
+ return CompletableFuture.completedFuture(currentTerm);
+ }
+ return waiters;
+ }
+
+ /**
+ * Handles leader election notification.
+ *
+ * <p>Transitions from {@link State#WAITING_FOR_LEADER} to {@link State#LEADER_AVAILABLE}
+ * if the new term is greater than the current term. Stale notifications (with term <= current)
+ * are ignored. Has no effect if the state machine has been stopped.
+ *
+ * @param leader The newly elected leader node.
+ * @param term The term of the new leader.
+ */
+ void onLeaderElected(InternalClusterNode leader, long term) {
+ CompletableFuture<Long> futureToComplete = null;
+
+ synchronized (this) {
Review Comment:
Resolved with internal `mutex` field object.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741233088
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // New waiter should immediately fail with the same exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone(), "New waiter should be immediately completed");
+ assertTrue(waiter2.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when no waiters are pending still marks state as stopped. */
+ @Test
+ void testStopWhenNoWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // Stop without any waiters - should not throw
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // Subsequent awaitLeader should fail
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when leader is available fails subsequent awaitLeader. */
+ @Test
+ void testStopWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // awaitLeader should fail after stop, even though leader was available
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Multiple stop calls are idempotent. */
+ @Test
+ void testMultipleStopCalls() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException exception1 = new RuntimeException("Shutdown 1");
+ RuntimeException exception2 = new RuntimeException("Shutdown 2");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(exception1);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Second stop call should be ignored
+ state.stop(exception2);
+
+ // Subsequent awaitLeader should fail with the FIRST exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(exception1, thrown.getCause(), "Should use exception from first stop call");
+ }
+
+ /** onLeaderElected is ignored after stop. */
+ @Test
+ void testOnLeaderElectedIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Leader election after stop should be ignored
+ state.onLeaderElected(leader, 5);
+
+ // State should still be stopped.
+ assertTrue(state.stopped());
+
+ // New waiter should still fail
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter2.isCompletedExceptionally());
+ }
+
+ /** onGroupUnavailable is ignored after stop. */
+ @Test
+ void testOnGroupUnavailableIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ // Group unavailable after stop should be ignored
+ state.onGroupUnavailable(1);
+
+ assertTrue(state.stopped());
+ }
+
+ /** Stop preserves exception type. */
+ @Test
+ void testStopPreservesExceptionType() {
Review Comment:
done
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // New waiter should immediately fail with the same exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone(), "New waiter should be immediately completed");
+ assertTrue(waiter2.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when no waiters are pending still marks state as stopped. */
+ @Test
+ void testStopWhenNoWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // Stop without any waiters - should not throw
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // Subsequent awaitLeader should fail
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop when leader is available fails subsequent awaitLeader. */
+ @Test
+ void testStopWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ assertTrue(state.stopped());
+
+ // awaitLeader should fail after stop, even though leader was available
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Multiple stop calls are idempotent. */
+ @Test
+ void testMultipleStopCalls() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException exception1 = new RuntimeException("Shutdown 1");
+ RuntimeException exception2 = new RuntimeException("Shutdown 2");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(exception1);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Second stop call should be ignored
+ state.stop(exception2);
+
+ // Subsequent awaitLeader should fail with the FIRST exception
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter2::join);
+ assertSame(exception1, thrown.getCause(), "Should use exception from first stop call");
+ }
+
+ /** onLeaderElected is ignored after stop. */
+ @Test
+ void testOnLeaderElectedIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ state.stop(testException);
+ assertTrue(waiter1.isCompletedExceptionally());
+
+ // Leader election after stop should be ignored
+ state.onLeaderElected(leader, 5);
+
+ // State should still be stopped.
+ assertTrue(state.stopped());
+
+ // New waiter should still fail
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter2.isCompletedExceptionally());
+ }
+
+ /** onGroupUnavailable is ignored after stop. */
+ @Test
+ void testOnGroupUnavailableIgnoredAfterStop() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ state.onLeaderElected(leader, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.stop(testException);
+
+ // Group unavailable after stop should be ignored
+ state.onGroupUnavailable(1);
+
+ assertTrue(state.stopped());
+ }
+
+ /** Stop preserves exception type. */
+ @Test
+ void testStopPreservesExceptionType() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ IllegalStateException testException = new IllegalStateException("Component stopping");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ state.stop(testException);
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertThat(thrown.getCause(), instanceOf(IllegalStateException.class));
+ assertEquals("Component stopping", thrown.getCause().getMessage());
+ }
+
+ /** stopped() returns false before stop and true after. */
+ @Test
+ void testStopped() {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "sk0x50 (via GitHub)" <gi...@apache.org>.
sk0x50 merged PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2740542026
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -81,11 +119,32 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
private final RaftGroupService raftClient;
/** Executor to invoke RPC requests. */
- private final Executor executor;
+ private final ScheduledExecutorService executor;
/** RAFT configuration. */
private final RaftConfiguration raftConfiguration;
+ /** State machine for tracking leader availability. */
+ private final LeaderAvailabilityState leaderAvailabilityState;
+
+ /** Command marshaller. */
+ private final Marshaller commandsMarshaller;
+
+ /** Throttling context holder. */
+ private final ThrottlingContextHolder throttlingContextHolder;
Review Comment:
removed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741275297
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741101740
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/RaftGroupServiceTest.java:
##########
@@ -680,6 +680,51 @@ public void testRetryOnErrorWithTimeout(RaftError error) {
assertThat(response, willThrow(TimeoutException.class, "Send with retry timed out"));
}
+ /**
+ * Tests that UNKNOWN/EINTERNAL/ENOENT errors retry on the same peer (the leader) for ActionRequests.
+ * This is because these errors are transient and the leader is likely to recover.
+ */
+ @ParameterizedTest
+ @EnumSource(names = {"UNKNOWN", "EINTERNAL", "ENOENT"})
+ public void testRetryOnTransientErrorRetriesOnSamePeer(RaftError error) {
+ Peer leaderPeer = NODES.get(0);
+
+ // First call returns error, second call succeeds - both to the same leader.
+ when(messagingService.invoke(
+ argThat((InternalClusterNode node) -> node != null && node.name().equals(leaderPeer.consistentId())),
+ any(ReadActionRequest.class),
+ anyLong())
+ )
+ .thenReturn(completedFuture(FACTORY.errorResponse()
+ .errorCode(error.getNumber())
+ .build()))
+ .thenReturn(completedFuture(FACTORY.actionResponse().result(null).build()));
+
+ RaftGroupService service = startRaftGroupServiceWithRefreshLeader(NODES);
+
+ assertThat(service.leader(), is(leaderPeer));
+
+ CompletableFuture<Object> response = service.run(mock(ReadCommand.class));
+
+ assertThat(response, willBe(nullValue()));
+
+ // Verify that only the leader was called (twice: once with error, once with success).
+ verify(messagingService, atLeastOnce()).invoke(
+ argThat((InternalClusterNode target) -> target != null && target.name().equals(leaderPeer.consistentId())),
+ any(ReadActionRequest.class),
+ anyLong()
+ );
+
+ // Verify that other peers were NOT called.
+ for (Peer otherPeer : NODES.subList(1, NODES.size())) {
+ verify(messagingService, org.mockito.Mockito.never()).invoke(
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741199555
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737314568
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
Review Comment:
Let's left as is for now. Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737294276
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
Review Comment:
Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2727774832
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Recoverable error during the request occurred (will be retried) [request={}, peer={}, newPeer={}, traceId={}].",
+ includeSensitive() ? retryContext.request() : retryContext.request().toStringForLightLogging(),
+ retryContext.targetPeer(),
+ nextPeer,
+ retryContext.errorTraceId()
+ );
+ }
+ }
+
+ /**
+ * Handles throwable in single attempt mode.
+ */
+ private void handleThrowableSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ Throwable err,
+ RetryContext retryContext
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ sendWithRetrySingleAttempt(fut, retryContext.nextAttemptForUnavailablePeer(nextPeer, shortReasonMessage));
+ }
+
+ /**
+ * Handles error response in single attempt mode.
+ *
+ * <p>In single-attempt mode, each peer is tried at most once. All errors mark the peer
+ * as tried and move to the next peer. When all peers have been tried, the request fails
+ * with {@link ReplicationGroupUnavailableException}.
+ */
+ private void handleErrorResponseSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ ErrorResponse resp,
+ RetryContext retryContext
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ // Single-attempt mode: ALWAYS mark current peer as unavailable, regardless of trackCurrentAs.
+ // This prevents retrying the same peer and matches original behavior where:
+ // - transient errors: would select new peer, mark current unavailable
+ // - peer unavailable: marks current unavailable
+ // - no leader: marks current unavailable (no NO_LEADER distinction needed)
+ // - leader redirect: marks current unavailable (to prevent redirect loops)
+ sendWithRetrySingleAttempt(fut, context.nextAttemptForUnavailablePeer(nextPeer, reason));
+ }
+
+ @Override
+ public void onAllPeersExhausted() {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+ }
+ };
+
+ handleErrorResponseCommon(fut, resp, retryContext, strategy, false);
+ }
+
+ /**
+ * Sends a request with retry and waits for leader on leader absence.
+ */
+ private void sendWithRetryWaitingForLeader(
+ CompletableFuture<ActionResponse> fut,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long requestStartTime = Utils.monotonicMs();
+ long stopTime = retryContext.stopTime();
+
+ if (requestStartTime >= stopTime) {
+ // Retry timeout expired - fail with timeout exception.
+ // For non-leader-related retriable errors, we don't wait for leader, just fail.
+ fut.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ ThrottlingContextHolder peerThrottlingContextHolder = throttlingContextHolder.peerContextHolder(
+ retryContext.targetPeer().consistentId()
+ );
+
+ peerThrottlingContextHolder.beforeRequest();
+ retryContext.onNewAttempt();
+
+ // Bound response timeout by remaining time until retry stop time.
+ long responseTimeout = Math.min(peerThrottlingContextHolder.peerRequestTimeoutMillis(), stopTime - requestStartTime);
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ // Enforce timeout even if messaging service doesn't (e.g., in tests with mocks).
+ .orTimeout(responseTimeout > 0 ? responseTimeout : Long.MAX_VALUE, TimeUnit.MILLISECONDS)
+ .whenComplete((resp, err) -> {
+ peerThrottlingContextHolder.afterRequest(requestStartTime, retriableError(err, resp));
+
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableWithLeaderWait(fut, err, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseWithLeaderWait(fut, (ErrorResponse) resp, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((ActionResponse) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Handles throwable in leader-wait mode.
+ */
+ private void handleThrowableWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ Throwable err,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ scheduleRetryWithLeaderWait(fut, retryContext.nextAttempt(nextPeer, shortReasonMessage), cmd, deadline, termWhenStarted);
+ }
+
+ /**
+ * Handles error response in leader-wait mode.
+ *
+ * <p>In leader-wait mode:
+ * <ul>
+ * <li>Transient errors (EBUSY/EAGAIN) retry on the same peer after delay</li>
+ * <li>"No leader" errors try each peer once, then wait for leader notification</li>
+ * <li>Peer unavailability errors cycle through peers until timeout</li>
+ * </ul>
+ */
+ private void handleErrorResponseWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ ErrorResponse resp,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ RetryContext nextContext;
+ if (trackCurrentAs == null) {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2726459695
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
Would we use for a request timeout only this "marker" value? If so, why do not pass `retryContext.responseTimeoutMillis()` explicitly?
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
Also should we worry about `sendWithRetryTimeout < responseTimeout` case?
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
Review Comment:
Kind of tastes, optional comment: I believe that then less indentations in a code then better, so approach like:
```java
if (!LOG.isDebugEnabled()) {
return;
}
LOG.debug(
"Recoverable error during the request occurred (will be retried) [request={}, peer={}, newPeer={}, traceId={}].",
includeSensitive() ? retryContext.request() : retryContext.request().toStringForLightLogging(),
retryContext.targetPeer(),
nextPeer,
retryContext.errorTraceId()
);
```
in my perspective will be better. Future us will have 4 more characters for the message in line. But as I said, you may omit this comment.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
Review Comment:
Shouldn't we place the annotation right before `Peer` return type that may be `null`?
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -81,11 +119,32 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
private final RaftGroupService raftClient;
/** Executor to invoke RPC requests. */
- private final Executor executor;
+ private final ScheduledExecutorService executor;
/** RAFT configuration. */
private final RaftConfiguration raftConfiguration;
+ /** State machine for tracking leader availability. */
+ private final LeaderAvailabilityState leaderAvailabilityState;
+
+ /** Command marshaller. */
+ private final Marshaller commandsMarshaller;
+
+ /** Throttling context holder. */
+ private final ThrottlingContextHolder throttlingContextHolder;
Review Comment:
I see no real usages for this object. Both usages are for throttling adaptive timeout as invoke response timeout. Yet not in production code, in all test usages there will be `-1` that is a "marker" to use this throttling timeout. So, we have some code that (yet) test-only. From the other hand we already have `raftConfiguration.responseTimeoutMillis().value()` for such case that may be easily changed for test purposes.
In my opinion, both, this class "timeout related retry approach" and "adaptive delaying throttling retry approach" should be implementation a retry strategy interfaces. But I believe this is a ticket about refactoring for future us. Now I think it's better to just get value from the raft configuration there.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
Review Comment:
There and below in the file.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Recoverable error during the request occurred (will be retried) [request={}, peer={}, newPeer={}, traceId={}].",
+ includeSensitive() ? retryContext.request() : retryContext.request().toStringForLightLogging(),
+ retryContext.targetPeer(),
+ nextPeer,
+ retryContext.errorTraceId()
+ );
+ }
+ }
+
+ /**
+ * Handles throwable in single attempt mode.
+ */
+ private void handleThrowableSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ Throwable err,
+ RetryContext retryContext
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ sendWithRetrySingleAttempt(fut, retryContext.nextAttemptForUnavailablePeer(nextPeer, shortReasonMessage));
+ }
+
+ /**
+ * Handles error response in single attempt mode.
+ *
+ * <p>In single-attempt mode, each peer is tried at most once. All errors mark the peer
+ * as tried and move to the next peer. When all peers have been tried, the request fails
+ * with {@link ReplicationGroupUnavailableException}.
+ */
+ private void handleErrorResponseSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ ErrorResponse resp,
+ RetryContext retryContext
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ // Single-attempt mode: ALWAYS mark current peer as unavailable, regardless of trackCurrentAs.
+ // This prevents retrying the same peer and matches original behavior where:
+ // - transient errors: would select new peer, mark current unavailable
+ // - peer unavailable: marks current unavailable
+ // - no leader: marks current unavailable (no NO_LEADER distinction needed)
+ // - leader redirect: marks current unavailable (to prevent redirect loops)
+ sendWithRetrySingleAttempt(fut, context.nextAttemptForUnavailablePeer(nextPeer, reason));
+ }
+
+ @Override
+ public void onAllPeersExhausted() {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+ }
+ };
+
+ handleErrorResponseCommon(fut, resp, retryContext, strategy, false);
+ }
+
+ /**
+ * Sends a request with retry and waits for leader on leader absence.
+ */
+ private void sendWithRetryWaitingForLeader(
+ CompletableFuture<ActionResponse> fut,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long requestStartTime = Utils.monotonicMs();
+ long stopTime = retryContext.stopTime();
+
+ if (requestStartTime >= stopTime) {
+ // Retry timeout expired - fail with timeout exception.
+ // For non-leader-related retriable errors, we don't wait for leader, just fail.
+ fut.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ ThrottlingContextHolder peerThrottlingContextHolder = throttlingContextHolder.peerContextHolder(
+ retryContext.targetPeer().consistentId()
+ );
+
+ peerThrottlingContextHolder.beforeRequest();
+ retryContext.onNewAttempt();
+
+ // Bound response timeout by remaining time until retry stop time.
+ long responseTimeout = Math.min(peerThrottlingContextHolder.peerRequestTimeoutMillis(), stopTime - requestStartTime);
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ // Enforce timeout even if messaging service doesn't (e.g., in tests with mocks).
+ .orTimeout(responseTimeout > 0 ? responseTimeout : Long.MAX_VALUE, TimeUnit.MILLISECONDS)
+ .whenComplete((resp, err) -> {
+ peerThrottlingContextHolder.afterRequest(requestStartTime, retriableError(err, resp));
+
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableWithLeaderWait(fut, err, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseWithLeaderWait(fut, (ErrorResponse) resp, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((ActionResponse) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Handles throwable in leader-wait mode.
+ */
+ private void handleThrowableWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ Throwable err,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ scheduleRetryWithLeaderWait(fut, retryContext.nextAttempt(nextPeer, shortReasonMessage), cmd, deadline, termWhenStarted);
+ }
+
+ /**
+ * Handles error response in leader-wait mode.
+ *
+ * <p>In leader-wait mode:
+ * <ul>
+ * <li>Transient errors (EBUSY/EAGAIN) retry on the same peer after delay</li>
+ * <li>"No leader" errors try each peer once, then wait for leader notification</li>
+ * <li>Peer unavailability errors cycle through peers until timeout</li>
+ * </ul>
+ */
+ private void handleErrorResponseWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ ErrorResponse resp,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
Review Comment:
This and there I believe we should have named implementations. Moreover, I believe that `RetryExecutionStrategy` will be useful in a future refactoring that I mentioned. Let's extract it as a separate interface?
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
Review Comment:
I think it's better to decouple validation and next peer selection to separate methods. On non recoverable throwable you will return `null`. About validation I know from the code and javadoc, but method name doesn't tell us about it.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
Review Comment:
`[groupId=" + groupId() + "].`
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
Review Comment:
This and there I'd to suggest return a _value_ to complete future, not to handle future inside in some curly way. Let's case study: `handleErrorResponseSingleAttempt` has a pretty complex logic, and in some cases we even don't want to complete the future (on transient error codes).
Also we may see that `handleSmErrorResponse` is a copy-paste mostly from the general raft client implementation. And it definitely feels like this methods shouldn't be `static` - they contain a logic to handle and the result should be a value to complete future there, not to pass it through the long `static` method chain.
If you ask me, it may be the common `ErrorResponseHandlerOnRetry` interface with `@Nullable Throwable handleErrorResponse(ErrorResponse, RetryContext)` method. The return value may be null in case non-exceptional null completeness of the future. WDYT?
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Recoverable error during the request occurred (will be retried) [request={}, peer={}, newPeer={}, traceId={}].",
+ includeSensitive() ? retryContext.request() : retryContext.request().toStringForLightLogging(),
+ retryContext.targetPeer(),
+ nextPeer,
+ retryContext.errorTraceId()
+ );
+ }
+ }
+
+ /**
+ * Handles throwable in single attempt mode.
+ */
+ private void handleThrowableSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ Throwable err,
+ RetryContext retryContext
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ sendWithRetrySingleAttempt(fut, retryContext.nextAttemptForUnavailablePeer(nextPeer, shortReasonMessage));
+ }
+
+ /**
+ * Handles error response in single attempt mode.
+ *
+ * <p>In single-attempt mode, each peer is tried at most once. All errors mark the peer
+ * as tried and move to the next peer. When all peers have been tried, the request fails
+ * with {@link ReplicationGroupUnavailableException}.
+ */
+ private void handleErrorResponseSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ ErrorResponse resp,
+ RetryContext retryContext
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ // Single-attempt mode: ALWAYS mark current peer as unavailable, regardless of trackCurrentAs.
+ // This prevents retrying the same peer and matches original behavior where:
+ // - transient errors: would select new peer, mark current unavailable
+ // - peer unavailable: marks current unavailable
+ // - no leader: marks current unavailable (no NO_LEADER distinction needed)
+ // - leader redirect: marks current unavailable (to prevent redirect loops)
+ sendWithRetrySingleAttempt(fut, context.nextAttemptForUnavailablePeer(nextPeer, reason));
+ }
+
+ @Override
+ public void onAllPeersExhausted() {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+ }
+ };
+
+ handleErrorResponseCommon(fut, resp, retryContext, strategy, false);
+ }
+
+ /**
+ * Sends a request with retry and waits for leader on leader absence.
+ */
+ private void sendWithRetryWaitingForLeader(
+ CompletableFuture<ActionResponse> fut,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long requestStartTime = Utils.monotonicMs();
+ long stopTime = retryContext.stopTime();
+
+ if (requestStartTime >= stopTime) {
+ // Retry timeout expired - fail with timeout exception.
+ // For non-leader-related retriable errors, we don't wait for leader, just fail.
+ fut.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ ThrottlingContextHolder peerThrottlingContextHolder = throttlingContextHolder.peerContextHolder(
+ retryContext.targetPeer().consistentId()
+ );
+
+ peerThrottlingContextHolder.beforeRequest();
+ retryContext.onNewAttempt();
+
+ // Bound response timeout by remaining time until retry stop time.
+ long responseTimeout = Math.min(peerThrottlingContextHolder.peerRequestTimeoutMillis(), stopTime - requestStartTime);
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ // Enforce timeout even if messaging service doesn't (e.g., in tests with mocks).
+ .orTimeout(responseTimeout > 0 ? responseTimeout : Long.MAX_VALUE, TimeUnit.MILLISECONDS)
+ .whenComplete((resp, err) -> {
+ peerThrottlingContextHolder.afterRequest(requestStartTime, retriableError(err, resp));
+
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableWithLeaderWait(fut, err, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseWithLeaderWait(fut, (ErrorResponse) resp, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((ActionResponse) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Handles throwable in leader-wait mode.
+ */
+ private void handleThrowableWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ Throwable err,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ scheduleRetryWithLeaderWait(fut, retryContext.nextAttempt(nextPeer, shortReasonMessage), cmd, deadline, termWhenStarted);
+ }
+
+ /**
+ * Handles error response in leader-wait mode.
+ *
+ * <p>In leader-wait mode:
+ * <ul>
+ * <li>Transient errors (EBUSY/EAGAIN) retry on the same peer after delay</li>
+ * <li>"No leader" errors try each peer once, then wait for leader notification</li>
+ * <li>Peer unavailability errors cycle through peers until timeout</li>
+ * </ul>
+ */
+ private void handleErrorResponseWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ ErrorResponse resp,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ RetryContext nextContext;
+ if (trackCurrentAs == null) {
Review Comment:
`@Nullable` enum value is a marker that there should be another one value. I think `COMMON` is the good one: track the current one as common peer. And if-else tree below will became a switch (do we allow assignation from switch statement? If yes, so `RetryContext nextContext = switch(trackCurrentAs) { .. }` looks good.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
Review Comment:
And complete the future outside of method. I'm afraid of too many implicit actions inside of the method.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -81,11 +119,32 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
private final RaftGroupService raftClient;
/** Executor to invoke RPC requests. */
- private final Executor executor;
+ private final ScheduledExecutorService executor;
/** RAFT configuration. */
private final RaftConfiguration raftConfiguration;
+ /** State machine for tracking leader availability. */
+ private final LeaderAvailabilityState leaderAvailabilityState;
+
+ /** Command marshaller. */
+ private final Marshaller commandsMarshaller;
+
+ /** Throttling context holder. */
+ private final ThrottlingContextHolder throttlingContextHolder;
Review Comment:
Actually that you've started with `RetryExecutionStrategy` but anonymous implementations on hot path (raft commands handling, even exceptional) is a bad way.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
Review Comment:
I think we had kind of utils methods for busy locks wrapping? Like those in `IgniteUtils` ones?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737301509
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
Review Comment:
Omitted and I guess it's fine. Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737284972
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
Still actual, you pass it in `startRetryPhase`, but not actually use it because you use throttled delay timeout in calculation:
```java
long responseTimeout = Math.min(peerThrottlingContextHolder.peerRequestTimeoutMillis(), stopTime - requestStartTime);
```
Looks strange.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2743023616
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
Resolved.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
Ah, some why I missed `min`. Resolved.
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -81,11 +119,32 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
private final RaftGroupService raftClient;
/** Executor to invoke RPC requests. */
- private final Executor executor;
+ private final ScheduledExecutorService executor;
/** RAFT configuration. */
private final RaftConfiguration raftConfiguration;
+ /** State machine for tracking leader availability. */
+ private final LeaderAvailabilityState leaderAvailabilityState;
+
+ /** Command marshaller. */
+ private final Marshaller commandsMarshaller;
+
+ /** Throttling context holder. */
+ private final ThrottlingContextHolder throttlingContextHolder;
Review Comment:
Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2743024139
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/RaftGroupServiceTest.java:
##########
@@ -680,6 +680,51 @@ public void testRetryOnErrorWithTimeout(RaftError error) {
assertThat(response, willThrow(TimeoutException.class, "Send with retry timed out"));
}
+ /**
+ * Tests that UNKNOWN/EINTERNAL/ENOENT errors retry on the same peer (the leader) for ActionRequests.
+ * This is because these errors are transient and the leader is likely to recover.
+ */
+ @ParameterizedTest
+ @EnumSource(names = {"UNKNOWN", "EINTERNAL", "ENOENT"})
+ public void testRetryOnTransientErrorRetriesOnSamePeer(RaftError error) {
+ Peer leaderPeer = NODES.get(0);
+
+ // First call returns error, second call succeeds - both to the same leader.
+ when(messagingService.invoke(
+ argThat((InternalClusterNode node) -> node != null && node.name().equals(leaderPeer.consistentId())),
+ any(ReadActionRequest.class),
+ anyLong())
+ )
+ .thenReturn(completedFuture(FACTORY.errorResponse()
+ .errorCode(error.getNumber())
+ .build()))
+ .thenReturn(completedFuture(FACTORY.actionResponse().result(null).build()));
+
+ RaftGroupService service = startRaftGroupServiceWithRefreshLeader(NODES);
+
+ assertThat(service.leader(), is(leaderPeer));
+
+ CompletableFuture<Object> response = service.run(mock(ReadCommand.class));
+
+ assertThat(response, willBe(nullValue()));
+
+ // Verify that only the leader was called (twice: once with error, once with success).
+ verify(messagingService, atLeastOnce()).invoke(
+ argThat((InternalClusterNode target) -> target != null && target.name().equals(leaderPeer.consistentId())),
+ any(ReadActionRequest.class),
+ anyLong()
+ );
+
+ // Verify that other peers were NOT called.
+ for (Peer otherPeer : NODES.subList(1, NODES.size())) {
+ verify(messagingService, org.mockito.Mockito.never()).invoke(
Review Comment:
Resolved.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
Review Comment:
Resolved.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
Review Comment:
Resolved.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
Review Comment:
Resolved.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
Review Comment:
Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737247552
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
Review Comment:
Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2743025509
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
Review Comment:
Resolved, the test was removed.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
Review Comment:
Resolved within `testStopCompletesMultipleWaiters`.
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ state.onLeaderElected(leader, 1);
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Both should be completed
+ assertTrue(future1.isDone());
+ assertTrue(future2.isDone());
+ }
+
+ /** Stop completes waiting futures exceptionally. */
+ @Test
+ void testStopCompletesExceptionally() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter.isDone());
+ assertTrue(waiter.isCompletedExceptionally());
+
+ CompletionException thrown = assertThrows(CompletionException.class, waiter::join);
+ assertSame(testException, thrown.getCause());
+ }
+
+ /** Stop completes multiple waiters exceptionally. */
+ @Test
+ void testStopCompletesMultipleWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ // All waiters share the same future
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.stop(testException);
+
+ assertTrue(waiter1.isCompletedExceptionally());
+ assertTrue(waiter2.isCompletedExceptionally());
+ assertTrue(waiter3.isCompletedExceptionally());
+ }
+
+ /** Stop causes subsequent awaitLeader to fail immediately. */
+ @Test
+ void testStopFailsSubsequentWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ RuntimeException testException = new RuntimeException("Test shutdown");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
Review Comment:
Resolved with the merge in `testStopCompletesMultipleWaiters`, confirming.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741112108
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
Review Comment:
Done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741190459
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
+
+ assertEquals(0, state.currentTerm());
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** Concurrent leader elections with different terms. */
+ @Test
+ void testConcurrentLeaderElections() throws Exception {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ int threadCount = 10;
+ CountDownLatch startLatch = new CountDownLatch(1);
+ CountDownLatch doneLatch = new CountDownLatch(threadCount);
+ AtomicLong maxTerm = new AtomicLong(-1);
+
+ Thread[] threads = new Thread[threadCount];
+ for (int i = 0; i < threadCount; i++) {
+ int term = i;
+ threads[i] = new Thread(() -> {
+ try {
+ startLatch.await();
+ InternalClusterNode leader = createNode("leader-" + term);
+ state.onLeaderElected(leader, term);
+ maxTerm.updateAndGet(current -> Math.max(current, term));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } finally {
+ doneLatch.countDown();
+ }
+ });
+ threads[i].start();
+ }
+
+ startLatch.countDown();
+ assertTrue(doneLatch.await(5, TimeUnit.SECONDS));
+
+ // The final term should be the maximum term that was set
+ assertEquals(threadCount - 1, state.currentTerm());
+ }
+
+ /** State machine handles rapid state transitions. */
+ @Test
+ void testRapidStateTransitions() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leader = createNode("leader");
+
+ for (int i = 0; i < 100; i++) {
+ state.onLeaderElected(leader, i);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(i, state.currentTerm());
+
+ state.onGroupUnavailable(i);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+ }
+
+ /** awaitLeader returns same future for multiple calls when waiting. */
+ @Test
+ void testAwaitLeaderReturnsSameFuture() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ CompletableFuture<Long> future2 = state.awaitLeader();
+
+ // Should return the same future instance when in WAITING_FOR_LEADER state
+ assertSame(future1, future2, "Should return same future instance when waiting");
+ }
+
+ /** awaitLeader returns new completed future for multiple calls when leader available. */
+ @Test
+ void testAwaitLeaderReturnsNewCompletedFuture() {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2727673072
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737307446
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
+ private Peer getNextPeerForRecoverableError(CompletableFuture<?> fut, Throwable err, RetryContext retryContext) {
+ err = unwrapCause(err);
+
+ if (!recoverable(err)) {
+ fut.completeExceptionally(err);
+
+ return null;
+ }
+
+ Peer nextPeer = randomNode(retryContext, false);
+
+ if (nextPeer == null) {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return null;
+ }
+
+ return nextPeer;
+ }
+
+ private static void logRecoverableError(RetryContext retryContext, Peer nextPeer) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Recoverable error during the request occurred (will be retried) [request={}, peer={}, newPeer={}, traceId={}].",
+ includeSensitive() ? retryContext.request() : retryContext.request().toStringForLightLogging(),
+ retryContext.targetPeer(),
+ nextPeer,
+ retryContext.errorTraceId()
+ );
+ }
+ }
+
+ /**
+ * Handles throwable in single attempt mode.
+ */
+ private void handleThrowableSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ Throwable err,
+ RetryContext retryContext
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ sendWithRetrySingleAttempt(fut, retryContext.nextAttemptForUnavailablePeer(nextPeer, shortReasonMessage));
+ }
+
+ /**
+ * Handles error response in single attempt mode.
+ *
+ * <p>In single-attempt mode, each peer is tried at most once. All errors mark the peer
+ * as tried and move to the next peer. When all peers have been tried, the request fails
+ * with {@link ReplicationGroupUnavailableException}.
+ */
+ private void handleErrorResponseSingleAttempt(
+ CompletableFuture<? extends NetworkMessage> fut,
+ ErrorResponse resp,
+ RetryContext retryContext
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ // Single-attempt mode: ALWAYS mark current peer as unavailable, regardless of trackCurrentAs.
+ // This prevents retrying the same peer and matches original behavior where:
+ // - transient errors: would select new peer, mark current unavailable
+ // - peer unavailable: marks current unavailable
+ // - no leader: marks current unavailable (no NO_LEADER distinction needed)
+ // - leader redirect: marks current unavailable (to prevent redirect loops)
+ sendWithRetrySingleAttempt(fut, context.nextAttemptForUnavailablePeer(nextPeer, reason));
+ }
+
+ @Override
+ public void onAllPeersExhausted() {
+ fut.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+ }
+ };
+
+ handleErrorResponseCommon(fut, resp, retryContext, strategy, false);
+ }
+
+ /**
+ * Sends a request with retry and waits for leader on leader absence.
+ */
+ private void sendWithRetryWaitingForLeader(
+ CompletableFuture<ActionResponse> fut,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long requestStartTime = Utils.monotonicMs();
+ long stopTime = retryContext.stopTime();
+
+ if (requestStartTime >= stopTime) {
+ // Retry timeout expired - fail with timeout exception.
+ // For non-leader-related retriable errors, we don't wait for leader, just fail.
+ fut.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ ThrottlingContextHolder peerThrottlingContextHolder = throttlingContextHolder.peerContextHolder(
+ retryContext.targetPeer().consistentId()
+ );
+
+ peerThrottlingContextHolder.beforeRequest();
+ retryContext.onNewAttempt();
+
+ // Bound response timeout by remaining time until retry stop time.
+ long responseTimeout = Math.min(peerThrottlingContextHolder.peerRequestTimeoutMillis(), stopTime - requestStartTime);
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ // Enforce timeout even if messaging service doesn't (e.g., in tests with mocks).
+ .orTimeout(responseTimeout > 0 ? responseTimeout : Long.MAX_VALUE, TimeUnit.MILLISECONDS)
+ .whenComplete((resp, err) -> {
+ peerThrottlingContextHolder.afterRequest(requestStartTime, retriableError(err, resp));
+
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableWithLeaderWait(fut, err, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseWithLeaderWait(fut, (ErrorResponse) resp, retryContext, cmd, deadline, termWhenStarted);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((ActionResponse) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Handles throwable in leader-wait mode.
+ */
+ private void handleThrowableWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ Throwable err,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ Peer nextPeer = getNextPeerForRecoverableError(fut, err, retryContext);
+
+ if (nextPeer == null) {
+ return;
+ }
+
+ logRecoverableError(retryContext, nextPeer);
+
+ String shortReasonMessage = "Peer " + retryContext.targetPeer().consistentId()
+ + " threw " + unwrapCause(err).getClass().getSimpleName();
+ scheduleRetryWithLeaderWait(fut, retryContext.nextAttempt(nextPeer, shortReasonMessage), cmd, deadline, termWhenStarted);
+ }
+
+ /**
+ * Handles error response in leader-wait mode.
+ *
+ * <p>In leader-wait mode:
+ * <ul>
+ * <li>Transient errors (EBUSY/EAGAIN) retry on the same peer after delay</li>
+ * <li>"No leader" errors try each peer once, then wait for leader notification</li>
+ * <li>Peer unavailability errors cycle through peers until timeout</li>
+ * </ul>
+ */
+ private void handleErrorResponseWithLeaderWait(
+ CompletableFuture<ActionResponse> fut,
+ ErrorResponse resp,
+ RetryContext retryContext,
+ Command cmd,
+ long deadline,
+ long termWhenStarted
+ ) {
+ RetryExecutionStrategy strategy = new RetryExecutionStrategy() {
+ @Override
+ public void executeRetry(RetryContext context, Peer nextPeer, @Nullable PeerTracking trackCurrentAs, String reason) {
+ RetryContext nextContext;
+ if (trackCurrentAs == null) {
Review Comment:
Resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737311786
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
Review Comment:
Consider as resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737290392
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
Review Comment:
Still actual, should we check this inequality
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "JAkutenshi (via GitHub)" <gi...@apache.org>.
JAkutenshi commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2737305810
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -81,11 +119,32 @@ public class PhysicalTopologyAwareRaftGroupService implements TimeAwareRaftGroup
private final RaftGroupService raftClient;
/** Executor to invoke RPC requests. */
- private final Executor executor;
+ private final ScheduledExecutorService executor;
/** RAFT configuration. */
private final RaftConfiguration raftConfiguration;
+ /** State machine for tracking leader availability. */
+ private final LeaderAvailabilityState leaderAvailabilityState;
+
+ /** Command marshaller. */
+ private final Marshaller commandsMarshaller;
+
+ /** Throttling context holder. */
+ private final ThrottlingContextHolder throttlingContextHolder;
Review Comment:
About strategy is resolved, but the question about `ThrottlingContextHolder` is still present.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2727801204
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/PhysicalTopologyAwareRaftGroupService.java:
##########
@@ -326,7 +397,823 @@ private static boolean recoverable(Throwable t) {
@Override
public <R> CompletableFuture<R> run(Command cmd, long timeoutMillis) {
- return raftClient.run(cmd, timeoutMillis);
+ // Normalize timeout: negative values mean infinite wait.
+ long effectiveTimeout = (timeoutMillis < 0) ? Long.MAX_VALUE : timeoutMillis;
+ // Wait for leader mode (bounded or infinite).
+ long deadline = Utils.monotonicMsAfter(effectiveTimeout);
+
+ return executeWithBusyLock(responseFuture -> {
+ if (effectiveTimeout == 0) {
+ tryAllPeersOnce(responseFuture, cmd);
+ } else {
+ startRetryPhase(responseFuture, cmd, deadline, leaderAvailabilityState.currentTerm());
+ }
+ });
+ }
+
+ /**
+ * Resolves initial target peer for a command execution.
+ *
+ * <p>Tries the known leader first, falling back to a random peer if no leader is known.
+ *
+ * @return Initial target peer, or {@code null}.
+ */
+ @Nullable
+ private Peer resolveInitialPeer() {
+ Peer targetPeer = leader;
+ if (targetPeer == null) {
+ targetPeer = randomNode(null, false);
+ }
+ return targetPeer;
+ }
+
+ /**
+ * Tries all peers once without waiting for leader.
+ *
+ * @param resultFuture Future that completes with the response, or fails with {@link ReplicationGroupUnavailableException} if no
+ * peer responds successfully.
+ * @param cmd The command to execute.
+ */
+ private void tryAllPeersOnce(CompletableFuture<ActionResponse> resultFuture, Command cmd) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ 0, // Single attempt - no retry timeout
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetrySingleAttempt(resultFuture, context);
+ }
+
+ /**
+ * Executes an action within busy lock and transforms the response.
+ */
+ private <R> CompletableFuture<R> executeWithBusyLock(Consumer<CompletableFuture<ActionResponse>> action) {
+ var responseFuture = new CompletableFuture<ActionResponse>();
+ var resultFuture = new CompletableFuture<R>();
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+
+ return resultFuture;
+ }
+
+ try {
+ action.accept(responseFuture);
+
+ // Transform ActionResponse to result type.
+ responseFuture.whenComplete((resp, err) -> {
+ if (err != null) {
+ resultFuture.completeExceptionally(err);
+ } else {
+ resultFuture.complete((R) resp.result());
+ }
+ });
+
+ return resultFuture;
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Applies deadline to a future using orTimeout.
+ *
+ * @param future The future to apply deadline to.
+ * @param deadline Deadline in monotonic milliseconds, or Long.MAX_VALUE for no deadline.
+ * @return The future with timeout applied, or the original future if no deadline.
+ */
+ private static <T> CompletableFuture<T> applyDeadline(CompletableFuture<T> future, long deadline) {
+ if (deadline == Long.MAX_VALUE) {
+ return future;
+ }
+ long remainingTime = deadline - Utils.monotonicMs();
+ if (remainingTime <= 0) {
+ return CompletableFuture.failedFuture(new TimeoutException());
+ }
+ return future.orTimeout(remainingTime, TimeUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a timeout exception for leader wait.
+ */
+ private ReplicationGroupUnavailableException createTimeoutException() {
+ return new ReplicationGroupUnavailableException(
+ groupId(),
+ "Timeout waiting for leader [groupId=" + groupId() + "]."
+ );
+ }
+
+ /**
+ * Waits for leader to become available and then retries the command.
+ */
+ private void waitForLeaderAndRetry(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline) {
+ CompletableFuture<Long> leaderFuture = leaderAvailabilityState.awaitLeader();
+
+ // Apply timeout if bounded.
+ CompletableFuture<Long> timedLeaderFuture = applyDeadline(leaderFuture, deadline);
+
+ timedLeaderFuture.whenCompleteAsync((term, waitError) -> {
+ if (waitError != null) {
+ Throwable cause = unwrapCause(waitError);
+ if (cause instanceof TimeoutException) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ } else {
+ resultFuture.completeExceptionally(cause);
+ }
+ return;
+ }
+
+ if (!busyLock.enterBusy()) {
+ resultFuture.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ // Leader is available, now run the command with retry logic.
+ startRetryPhase(resultFuture, cmd, deadline, term);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }, executor);
+ }
+
+ /**
+ * Starts the retry phase after leader is available.
+ */
+ private void startRetryPhase(CompletableFuture<ActionResponse> resultFuture, Command cmd, long deadline, long term) {
+ Peer targetPeer = resolveInitialPeer();
+ if (targetPeer == null) {
+ resultFuture.completeExceptionally(new ReplicationGroupUnavailableException(groupId()));
+
+ return;
+ }
+
+ // Check deadline before starting retry phase.
+ long now = Utils.monotonicMs();
+ if (deadline != Long.MAX_VALUE && now >= deadline) {
+ resultFuture.completeExceptionally(createTimeoutException());
+ return;
+ }
+
+ // Use retry timeout bounded by remaining time until deadline.
+ long configTimeout = raftConfiguration.retryTimeoutMillis().value();
+
+ long sendWithRetryTimeoutMillis = deadline == Long.MAX_VALUE ? configTimeout : Math.min(configTimeout, deadline - now);
+
+ var context = new RetryContext(
+ groupId().toString(),
+ targetPeer,
+ cmd::toStringForLightLogging,
+ createRequestFactory(cmd),
+ sendWithRetryTimeoutMillis,
+ RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ );
+
+ sendWithRetryWaitingForLeader(resultFuture, context, cmd, deadline, term);
+ }
+
+ /**
+ * Creates a request factory for the given command.
+ */
+ private Function<Peer, ActionRequest> createRequestFactory(Command cmd) {
+ if (cmd instanceof WriteCommand) {
+ return targetPeer -> MESSAGES_FACTORY.writeActionRequest()
+ .groupId(groupId().toString())
+ .command(commandsMarshaller.marshall(cmd))
+ .deserializedCommand((WriteCommand) cmd)
+ .build();
+ } else {
+ return targetPeer -> MESSAGES_FACTORY.readActionRequest()
+ .groupId(groupId().toString())
+ .command((ReadCommand) cmd)
+ .readOnlySafe(true)
+ .build();
+ }
+ }
+
+ /**
+ * Sends a request with single attempt (no retry on timeout).
+ *
+ * <p>In single-attempt mode, each peer is tried at most once.
+ */
+ private <R extends NetworkMessage> void sendWithRetrySingleAttempt(
+ CompletableFuture<R> fut,
+ RetryContext retryContext
+ ) {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ long responseTimeout = retryContext.responseTimeoutMillis() == RetryContext.USE_DEFAULT_RESPONSE_TIMEOUT
+ ? throttlingContextHolder.peerRequestTimeoutMillis() : retryContext.responseTimeoutMillis();
+
+ retryContext.onNewAttempt();
+
+ resolvePeer(retryContext.targetPeer())
+ .thenCompose(node -> clusterService.messagingService().invoke(node, retryContext.request(), responseTimeout))
+ .whenComplete((resp, err) -> {
+ if (!busyLock.enterBusy()) {
+ fut.completeExceptionally(stoppingExceptionFactory.create("Raft client is stopping [" + groupId() + "]."));
+ return;
+ }
+
+ try {
+ if (err != null) {
+ handleThrowableSingleAttempt(fut, err, retryContext);
+ } else if (resp instanceof ErrorResponse) {
+ handleErrorResponseSingleAttempt(fut, (ErrorResponse) resp, retryContext);
+ } else if (resp instanceof SMErrorResponse) {
+ handleSmErrorResponse(fut, (SMErrorResponse) resp, retryContext);
+ } else {
+ leader = retryContext.targetPeer();
+ fut.complete((R) resp);
+ }
+ } catch (Throwable e) {
+ fut.completeExceptionally(e);
+ } finally {
+ busyLock.leaveBusy();
+ }
+ });
+ } finally {
+ busyLock.leaveBusy();
+ }
+ }
+
+ /**
+ * Validates throwable and returns next peer for retry.
+ *
+ * <p>If the error is not recoverable or no peers are available, completes the future exceptionally and returns null.
+ *
+ * @param fut Future to complete exceptionally if retry is not possible.
+ * @param err The throwable to check.
+ * @param retryContext Retry context for getting next peer.
+ * @return Next peer for retry, or null if the future was already completed with an error.
+ */
+ @Nullable
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2740619912
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741168311
##########
modules/raft/src/test/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityStateTest.java:
##########
@@ -0,0 +1,516 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import static org.apache.ignite.internal.testframework.IgniteTestUtils.deriveUuidFrom;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.internal.network.ClusterNodeImpl;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.apache.ignite.internal.raft.client.LeaderAvailabilityState.State;
+import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
+import org.apache.ignite.network.NetworkAddress;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LeaderAvailabilityState} state machine.
+ */
+public class LeaderAvailabilityStateTest extends BaseIgniteAbstractTest {
+
+ /** Initial state should be WAITING_FOR_LEADER with term -1. */
+ @Test
+ void testInitialState() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ assertEquals(-1, state.currentTerm());
+ }
+
+ /** awaitLeader returns incomplete future when no leader elected. */
+ @Test
+ void testAwaitLeaderWhenNoLeader() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertFalse(future.isDone(), "Future should not be completed when no leader");
+ }
+
+ /** awaitLeader returns completed future when leader is available. */
+ @Test
+ void testAwaitLeaderWhenLeaderAvailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ CompletableFuture<Long> future = state.awaitLeader();
+
+ assertTrue(future.isDone(), "Future should be completed when leader available");
+ assertEquals(1L, future.join());
+ }
+
+ /** onLeaderElected transitions to LEADER_AVAILABLE state. */
+ @Test
+ void testOnLeaderElectedTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(1, state.currentTerm());
+ }
+
+ /** onLeaderElected completes waiting futures. */
+ @Test
+ void testOnLeaderElectedCompletesWaiters() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter = state.awaitLeader();
+ assertFalse(waiter.isDone());
+
+ state.onLeaderElected(leaderNode, 5);
+
+ assertTrue(waiter.isDone());
+ assertEquals(5L, waiter.join());
+ }
+
+ /** Stale term notifications are ignored. */
+ @Test
+ void testStaleTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Stale notification with lower term should be ignored
+ state.onLeaderElected(leaderNode2, 3);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Equal term notifications are ignored. */
+ @Test
+ void testEqualTermIgnored() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+
+ // Same term should be ignored
+ state.onLeaderElected(leaderNode, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** Higher term updates current term. */
+ @Test
+ void testHigherTermUpdates() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode1 = createNode("leader-1");
+ InternalClusterNode leaderNode2 = createNode("leader-2");
+
+ state.onLeaderElected(leaderNode1, 1);
+ assertEquals(1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode2, 5);
+ assertEquals(5, state.currentTerm());
+ }
+
+ /** onGroupUnavailable transitions back to WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableTransition() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+
+ state.onGroupUnavailable(1);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** onGroupUnavailable ignored if term changed. */
+ @Test
+ void testOnGroupUnavailableIgnoredIfTermChanged() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ state.onLeaderElected(leaderNode, 2);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ assertEquals(2, state.currentTerm());
+
+ // Should be ignored because term changed from 1 to 2
+ state.onGroupUnavailable(1);
+ assertEquals(State.LEADER_AVAILABLE, state.currentState());
+ }
+
+ /** onGroupUnavailable has no effect when already in WAITING_FOR_LEADER. */
+ @Test
+ void testOnGroupUnavailableWhenAlreadyWaiting() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+
+ // Already in WAITING_FOR_LEADER, should have no effect
+ state.onGroupUnavailable(0);
+ assertEquals(State.WAITING_FOR_LEADER, state.currentState());
+ }
+
+ /** After onGroupUnavailable, new waiters get fresh future. */
+ @Test
+ void testNewWaitersAfterUnavailable() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ state.onLeaderElected(leaderNode, 1);
+ CompletableFuture<Long> future1 = state.awaitLeader();
+ assertTrue(future1.isDone());
+
+ state.onGroupUnavailable(1);
+
+ CompletableFuture<Long> future2 = state.awaitLeader();
+ assertFalse(future2.isDone(), "New waiter should get incomplete future after unavailable");
+ }
+
+ /** Multiple waiters are all completed on leader election. */
+ @Test
+ void testMultipleWaitersCompleted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader-node");
+
+ CompletableFuture<Long> waiter1 = state.awaitLeader();
+ CompletableFuture<Long> waiter2 = state.awaitLeader();
+ CompletableFuture<Long> waiter3 = state.awaitLeader();
+
+ assertFalse(waiter1.isDone());
+ assertFalse(waiter2.isDone());
+ assertFalse(waiter3.isDone());
+
+ state.onLeaderElected(leaderNode, 10);
+
+ assertTrue(waiter1.isDone());
+ assertTrue(waiter2.isDone());
+ assertTrue(waiter3.isDone());
+
+ assertEquals(10L, waiter1.join());
+ assertEquals(10L, waiter2.join());
+ assertEquals(10L, waiter3.join());
+ }
+
+ /** Term 0 is accepted as first valid term. */
+ @Test
+ void testTermZeroAccepted() {
+ LeaderAvailabilityState state = new LeaderAvailabilityState();
+ InternalClusterNode leaderNode = createNode("leader");
+
+ assertEquals(-1, state.currentTerm());
+
+ state.onLeaderElected(leaderNode, 0);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] IGNITE-27156 Change timeout semantics [ignite-3]
Posted by "Cyrill (via GitHub)" <gi...@apache.org>.
Cyrill commented on code in PR #7242:
URL: https://github.com/apache/ignite-3/pull/7242#discussion_r2741249382
##########
modules/raft/src/main/java/org/apache/ignite/internal/raft/client/LeaderAvailabilityState.java:
##########
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.raft.client;
+
+import java.util.concurrent.CompletableFuture;
+import org.apache.ignite.internal.logger.IgniteLogger;
+import org.apache.ignite.internal.logger.Loggers;
+import org.apache.ignite.internal.network.InternalClusterNode;
+import org.jetbrains.annotations.VisibleForTesting;
+
+/**
+ * State machine for tracking leader availability.
+ *
+ * <p>Manages transitions between two states:
+ * <ul>
+ * <li>{@link State#WAITING_FOR_LEADER} - No leader is currently known, operations must wait</li>
+ * <li>{@link State#LEADER_AVAILABLE} - A leader is known, operations can proceed</li>
+ * </ul>
+ *
+ * <p>State transitions:
+ * <pre>
+ * WAITING_FOR_LEADER --[onLeaderElected]--> LEADER_AVAILABLE
+ * LEADER_AVAILABLE --[onGroupUnavailable]--> WAITING_FOR_LEADER
+ * Any state --[stop]--> stopped (terminal state)
+ * </pre>
+ *
+ * <p>Thread-safe: all public methods are synchronized or use proper synchronization.
+ */
+@VisibleForTesting
+class LeaderAvailabilityState {
+ /** The logger. */
+ private static final IgniteLogger LOG = Loggers.forClass(LeaderAvailabilityState.class);
+
+ /** Possible states of leader availability. */
+ enum State {
+ /** No leader is known, waiting for leader election. */
+ WAITING_FOR_LEADER,
+ /** A leader is available and operations can proceed. */
+ LEADER_AVAILABLE
+ }
+
+ /** Current state. Guarded by {@code this}. */
+ private State currentState = State.WAITING_FOR_LEADER;
+
+ /** Current leader term. Initialized to -1 to accept term 0 as the first valid term. Guarded by {@code this}. */
+ private long currentTerm = -1;
+
+ /** Future that waiters block on. Guarded by {@code this}. */
+ private CompletableFuture<Long> waiters = new CompletableFuture<>();
+
+ /** Whether the state machine has been stopped. Guarded by {@code this}. */
+ private boolean stopped = false;
+
+ /** Exception used to fail futures after destruction. Guarded by {@code this}. */
+ private Throwable stopException;
+
+ /**
+ * Returns a future that completes when a leader becomes available.
+ *
+ * <p>If a leader is already available, returns an already-completed future with the current term.
+ * Otherwise, returns a future that will complete when {@link #onLeaderElected} is called.
+ * If the state machine has been destroyed, returns a failed future.
+ *
+ * @return Future that completes with the leader term when a leader is available.
+ */
+ synchronized CompletableFuture<Long> awaitLeader() {
+ if (stopped) {
+ return CompletableFuture.failedFuture(stopException);
+ }
+ if (currentState == State.LEADER_AVAILABLE) {
+ return CompletableFuture.completedFuture(currentTerm);
+ }
+ return waiters;
+ }
+
+ /**
+ * Handles leader election notification.
+ *
+ * <p>Transitions from {@link State#WAITING_FOR_LEADER} to {@link State#LEADER_AVAILABLE}
+ * if the new term is greater than the current term. Stale notifications (with term <= current)
+ * are ignored. Has no effect if the state machine has been stopped.
+ *
+ * @param leader The newly elected leader node.
+ * @param term The term of the new leader.
+ */
+ void onLeaderElected(InternalClusterNode leader, long term) {
+ CompletableFuture<Long> futureToComplete = null;
+
+ synchronized (this) {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: notifications-unsubscribe@ignite.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org