You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2026/08/12 12:43:30 UTC

(fluss) branch main updated: [server] Fix ISR shrink race during leader transition (#3920)

This is an automated email from the ASF dual-hosted git repository.

wuchong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git


The following commit(s) were added to refs/heads/main by this push:
     new b2c511c18 [server] Fix ISR shrink race during leader transition (#3920)
b2c511c18 is described below

commit b2c511c18b5e8960ec6ee884fa46b70cc3bbd6dc
Author: hanliu0830 <ha...@gmail.com>
AuthorDate: Wed Aug 12 20:43:26 2026 +0800

    [server] Fix ISR shrink race during leader transition (#3920)
---
 .../coordinator/CoordinatorEventProcessor.java     |  24 +++++
 .../org/apache/fluss/server/replica/Replica.java   |  49 +++++-----
 .../coordinator/CoordinatorEventProcessorTest.java | 105 +++++++++++++++++++++
 .../apache/fluss/server/replica/AdjustIsrTest.java |  92 ++++++++++++++++++
 4 files changed, 247 insertions(+), 23 deletions(-)

diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index e879a1586..96220fee1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -2035,6 +2035,30 @@ public class CoordinatorEventProcessor implements EventProcessor {
                 throw new InvalidUpdateVersionException(
                         "The request bucket epoch in adjust isr request is lower than current bucket epoch in coordinator.");
             } else {
+                if (newLeaderAndIsr.leader() != currentLeaderAndIsr.leader()) {
+                    String errorMsg =
+                            String.format(
+                                    "Rejecting adjustIsr request for table bucket %s because request leader %s "
+                                            + "does not match current leader %s",
+                                    tableBucket,
+                                    newLeaderAndIsr.leader(),
+                                    currentLeaderAndIsr.leader());
+                    LOG.error(errorMsg);
+                    throw new FencedLeaderEpochException(errorMsg);
+                }
+
+                if (!newLeaderAndIsr.isr().contains(currentLeaderAndIsr.leader())) {
+                    String errorMsg =
+                            String.format(
+                                    "Rejecting adjustIsr request for table bucket %s because leader %s "
+                                            + "is not in the new ISR %s",
+                                    tableBucket,
+                                    currentLeaderAndIsr.leader(),
+                                    newLeaderAndIsr.isr());
+                    LOG.error(errorMsg);
+                    throw new IneligibleReplicaException(errorMsg);
+                }
+
                 // Check if the new ISR are all ineligible replicas (doesn't contain any shutting
                 // down tabletServers).
                 Set<Integer> ineligibleReplicas = new HashSet<>(newLeaderAndIsr.isr());
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index 5192cdeb0..784c4ad04 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -135,7 +135,6 @@ import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.IntSupplier;
 import java.util.function.Supplier;
@@ -191,7 +190,7 @@ public final class Replica {
     private final LogFormat logFormat;
     private final ArrowCompressionInfo arrowCompressionInfo;
     private final AtomicReference<Integer> leaderReplicaIdOpt = new AtomicReference<>();
-    private final ReadWriteLock leaderIsrUpdateLock = new ReentrantReadWriteLock();
+    private final ReentrantReadWriteLock leaderIsrUpdateLock = new ReentrantReadWriteLock();
     private final Clock clock;
     private final RemoteLogManager remoteLogManager;
 
@@ -1865,25 +1864,25 @@ public final class Replica {
      * <p>This function can be triggered when a replica's LEO has incremented.
      */
     private void maybeExpandISr(FollowerReplica followerReplica) {
-        IsrState currentIsrState = isrState;
         boolean needsIsrUpdate =
-                !currentIsrState.isInflight()
-                        && inReadLock(leaderIsrUpdateLock, () -> needsExpandIsr(followerReplica));
+                inReadLock(
+                        leaderIsrUpdateLock,
+                        () -> !isrState.isInflight() && needsExpandIsr(followerReplica));
 
         if (needsIsrUpdate) {
             Optional<IsrState.PendingExpandIsrState> adjustIsrUpdateOpt =
                     inWriteLock(
                             leaderIsrUpdateLock,
                             () -> {
+                                IsrState currentIsrState = isrState;
                                 // check if this replica needs to be added to the ISR.
-                                if (currentIsrState instanceof IsrState.CommittedIsrState) {
-                                    if (needsExpandIsr(followerReplica)) {
-                                        return Optional.of(
-                                                prepareIsrExpand(
-                                                        (IsrState.CommittedIsrState)
-                                                                currentIsrState,
-                                                        followerReplica.getFollowerId()));
-                                    }
+                                if (isLeader()
+                                        && currentIsrState instanceof IsrState.CommittedIsrState
+                                        && needsExpandIsr(followerReplica)) {
+                                    return Optional.of(
+                                            prepareIsrExpand(
+                                                    (IsrState.CommittedIsrState) currentIsrState,
+                                                    followerReplica.getFollowerId()));
                                 }
 
                                 return Optional.empty();
@@ -1897,28 +1896,27 @@ public final class Replica {
     }
 
     void maybeShrinkIsr() {
-        IsrState currentIstState = isrState;
         boolean needsIsrUpdate =
-                !currentIstState.isInflight()
-                        && inReadLock(leaderIsrUpdateLock, this::needsShrinkIsr);
+                inReadLock(leaderIsrUpdateLock, () -> !isrState.isInflight() && needsShrinkIsr());
 
         if (needsIsrUpdate) {
             Optional<IsrState.PendingShrinkIsrState> adjustIsrUpdateOpt =
                     inWriteLock(
                             leaderIsrUpdateLock,
                             () -> {
-                                if (isLeader()) {
+                                IsrState currentIsrState = isrState;
+                                if (isLeader()
+                                        && currentIsrState instanceof IsrState.CommittedIsrState) {
                                     List<Integer> outOfSyncFollowerReplicas =
                                             getOutOfSyncFollowerReplicas(replicaMaxLagTime);
-                                    if (currentIstState instanceof IsrState.CommittedIsrState
-                                            && !outOfSyncFollowerReplicas.isEmpty()) {
+                                    if (!outOfSyncFollowerReplicas.isEmpty()) {
                                         List<Integer> newIsr =
-                                                new ArrayList<>(currentIstState.isr());
+                                                new ArrayList<>(currentIsrState.isr());
                                         newIsr.removeAll(outOfSyncFollowerReplicas);
                                         LOG.info(
                                                 "Shrink ISR From {} to {} for bucket {}. Leader: (high watermark: {}, "
                                                         + "end offset: {}, out of sync replicas: {})",
-                                                currentIstState.isr(),
+                                                currentIsrState.isr(),
                                                 newIsr,
                                                 tableBucket,
                                                 logTablet.getHighWatermark(),
@@ -1927,7 +1925,7 @@ public final class Replica {
                                         return Optional.of(
                                                 prepareIsrShrink(
                                                         (IsrState.CommittedIsrState)
-                                                                currentIstState,
+                                                                currentIsrState,
                                                         newIsr,
                                                         outOfSyncFollowerReplicas));
                                     }
@@ -1950,7 +1948,7 @@ public final class Replica {
         // reflect the updated ISR even if there is a delay before we receive the confirmation.
         // Alternatively, if the update fails, no harm is done since the expanded ISR puts
         // a stricter requirement for advancement of the HW.
-        List<Integer> isrToSend = new ArrayList<>(isrState.isr());
+        List<Integer> isrToSend = new ArrayList<>(currentState.isr());
         isrToSend.add(newInSyncReplicaId);
 
         // TODO add server epoch to isr.
@@ -2365,6 +2363,11 @@ public final class Replica {
         return bucketEpoch;
     }
 
+    @VisibleForTesting
+    ReentrantReadWriteLock getLeaderIsrUpdateLock() {
+        return leaderIsrUpdateLock;
+    }
+
     @VisibleForTesting
     public List<Integer> getIsr() {
         return isrState.isr();
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
index cee0010fc..0bbb40b5c 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
@@ -1273,6 +1273,70 @@ class CoordinatorEventProcessorTest {
         assertThat(resultForBucketMap.values()).allMatch(AdjustIsrResultForBucket::succeeded);
     }
 
+    @Test
+    void testProcessAdjustIsrRejectsRequestFromNonLeader() throws Exception {
+        Map.Entry<TableBucket, LeaderAndIsr> entry =
+                createTableAndGetLeaderAndIsr("adjust_isr_from_non_leader");
+        TableBucket tableBucket = entry.getKey();
+        LeaderAndIsr currentLeaderAndIsr = entry.getValue();
+        int nonLeader =
+                currentLeaderAndIsr.isr().stream()
+                        .filter(replica -> replica != currentLeaderAndIsr.leader())
+                        .findFirst()
+                        .get();
+        LeaderAndIsr invalidLeaderAndIsr =
+                new LeaderAndIsr(
+                        nonLeader,
+                        currentLeaderAndIsr.leaderEpoch(),
+                        currentLeaderAndIsr.isr(),
+                        currentLeaderAndIsr.standbyReplicas(),
+                        currentLeaderAndIsr.coordinatorEpoch(),
+                        currentLeaderAndIsr.bucketEpoch());
+
+        AdjustIsrResultForBucket invalidResult = submitAdjustIsr(tableBucket, invalidLeaderAndIsr);
+
+        assertThat(invalidResult.getError().error())
+                .isEqualTo(Errors.FENCED_LEADER_EPOCH_EXCEPTION);
+        assertThat(invalidResult.getErrorMessage())
+                .contains(
+                        String.format(
+                                "request leader %s does not match current leader %s",
+                                nonLeader, currentLeaderAndIsr.leader()));
+        Optional<LeaderAndIsr> storedLeaderAndIsr =
+                fromCtx(ctx -> ctx.getBucketLeaderAndIsr(tableBucket));
+        assertThat(storedLeaderAndIsr).contains(currentLeaderAndIsr);
+    }
+
+    @Test
+    void testProcessAdjustIsrRejectsIsrWithoutCurrentLeader() throws Exception {
+        Map.Entry<TableBucket, LeaderAndIsr> entry =
+                createTableAndGetLeaderAndIsr("adjust_isr_without_current_leader");
+        TableBucket tableBucket = entry.getKey();
+        LeaderAndIsr currentLeaderAndIsr = entry.getValue();
+        List<Integer> isrWithoutLeader =
+                currentLeaderAndIsr.isr().stream()
+                        .filter(replica -> replica != currentLeaderAndIsr.leader())
+                        .collect(Collectors.toList());
+        LeaderAndIsr invalidLeaderAndIsr =
+                new LeaderAndIsr(
+                        currentLeaderAndIsr.leader(),
+                        currentLeaderAndIsr.leaderEpoch(),
+                        isrWithoutLeader,
+                        currentLeaderAndIsr.standbyReplicas(),
+                        currentLeaderAndIsr.coordinatorEpoch(),
+                        currentLeaderAndIsr.bucketEpoch());
+
+        AdjustIsrResultForBucket invalidResult = submitAdjustIsr(tableBucket, invalidLeaderAndIsr);
+        assertThat(invalidResult.getError().error()).isEqualTo(Errors.INELIGIBLE_REPLICA_EXCEPTION);
+        assertThat(invalidResult.getErrorMessage())
+                .contains(
+                        String.format(
+                                "leader %s is not in the new ISR", currentLeaderAndIsr.leader()));
+        Optional<LeaderAndIsr> storedLeaderAndIsr =
+                fromCtx(ctx -> ctx.getBucketLeaderAndIsr(tableBucket));
+        assertThat(storedLeaderAndIsr).contains(currentLeaderAndIsr);
+    }
+
     @Test
     void testDiskWriteLockedNotifyLeaderResponseMarksReplicaOffline() throws Exception {
         initCoordinatorChannel();
@@ -2497,6 +2561,47 @@ class CoordinatorEventProcessorTest {
         return event.getResultFuture().get(30, TimeUnit.SECONDS);
     }
 
+    private Map.Entry<TableBucket, LeaderAndIsr> createTableAndGetLeaderAndIsr(String tableName)
+            throws Exception {
+        initCoordinatorChannel();
+        TableAssignment tableAssignment =
+                generateAssignment(
+                        N_BUCKETS,
+                        REPLICATION_FACTOR,
+                        new TabletServerInfo[] {
+                            new TabletServerInfo(0, "rack0"),
+                            new TabletServerInfo(1, "rack1"),
+                            new TabletServerInfo(2, "rack2")
+                        });
+        long tableId =
+                metadataManager.createTable(
+                        TablePath.of(defaultDatabase, tableName),
+                        remoteDataDir,
+                        TEST_TABLE,
+                        tableAssignment,
+                        false);
+        verifyTableCreated(tableId, tableAssignment, N_BUCKETS, REPLICATION_FACTOR);
+
+        Map<TableBucket, LeaderAndIsr> bucketLeaderAndIsrMap =
+                new HashMap<>(
+                        waitValue(
+                                () -> fromCtx(ctx -> Optional.of(ctx.bucketLeaderAndIsr())),
+                                Duration.ofMinutes(1),
+                                "leader not elected"));
+        return bucketLeaderAndIsrMap.entrySet().iterator().next();
+    }
+
+    private AdjustIsrResultForBucket submitAdjustIsr(
+            TableBucket tableBucket, LeaderAndIsr newLeaderAndIsr) throws Exception {
+        CompletableFuture<AdjustIsrResponse> response = new CompletableFuture<>();
+        eventProcessor
+                .getCoordinatorEventManager()
+                .put(
+                        new AdjustIsrReceivedEvent(
+                                Collections.singletonMap(tableBucket, newLeaderAndIsr), response));
+        return getAdjustIsrResponseData(response.get()).get(tableBucket);
+    }
+
     private long createTable(TablePath tablePath, TabletServerInfo[] servers) {
         TableAssignment tableAssignment =
                 generateAssignment(N_BUCKETS, REPLICATION_FACTOR, servers);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java
index 7a5459c3f..77e425bbc 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java
@@ -35,9 +35,14 @@ import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 import static org.apache.fluss.record.TestData.DATA1;
+import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
 import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
 import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
 import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
@@ -126,6 +131,82 @@ public class AdjustIsrTest extends ReplicaTestBase {
         assertThat(replica.getIsr()).containsExactlyInAnyOrder(1);
     }
 
+    @Test
+    void testShrinkIsrUsesLatestStateAfterLeaderChange() throws Exception {
+        TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
+        Replica replica = makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, tb);
+        NotifyLeaderAndIsrData leaderData =
+                new NotifyLeaderAndIsrData(
+                        DATA1_PHYSICAL_TABLE_PATH,
+                        tb,
+                        Arrays.asList(1, 2),
+                        new LeaderAndIsr(1, 0, Arrays.asList(1, 2), Collections.emptyList(), 0, 0));
+
+        ReentrantReadWriteLock leaderIsrUpdateLock = replica.getLeaderIsrUpdateLock();
+
+        ExecutorService executor = Executors.newFixedThreadPool(3);
+        try {
+            AtomicReference<Thread> makeLeaderThread = new AtomicReference<>();
+            AtomicReference<Thread> advanceClockThread = new AtomicReference<>();
+            AtomicReference<Thread> shrinkIsrThread = new AtomicReference<>();
+
+            leaderIsrUpdateLock.writeLock().lock();
+            CompletableFuture<Void> makeLeaderFuture;
+            CompletableFuture<Void> advanceClockFuture;
+            CompletableFuture<Void> shrinkIsrFuture;
+            try {
+                makeLeaderFuture =
+                        CompletableFuture.runAsync(
+                                () -> {
+                                    makeLeaderThread.set(Thread.currentThread());
+                                    try {
+                                        replica.makeLeader(leaderData);
+                                    } catch (Exception e) {
+                                        throw new RuntimeException(e);
+                                    }
+                                },
+                                executor);
+                waitUntilQueued(leaderIsrUpdateLock, makeLeaderThread);
+
+                advanceClockFuture =
+                        CompletableFuture.runAsync(
+                                () -> {
+                                    advanceClockThread.set(Thread.currentThread());
+                                    leaderIsrUpdateLock.writeLock().lock();
+                                    try {
+                                        manualClock.advanceTime(
+                                                conf.get(ConfigOptions.LOG_REPLICA_MAX_LAG_TIME)
+                                                                .toMillis()
+                                                        + 1,
+                                                TimeUnit.MILLISECONDS);
+                                    } finally {
+                                        leaderIsrUpdateLock.writeLock().unlock();
+                                    }
+                                },
+                                executor);
+                waitUntilQueued(leaderIsrUpdateLock, advanceClockThread);
+
+                shrinkIsrFuture =
+                        CompletableFuture.runAsync(
+                                () -> {
+                                    shrinkIsrThread.set(Thread.currentThread());
+                                    replica.maybeShrinkIsr();
+                                },
+                                executor);
+                waitUntilQueued(leaderIsrUpdateLock, shrinkIsrThread);
+            } finally {
+                leaderIsrUpdateLock.writeLock().unlock();
+            }
+
+            makeLeaderFuture.get(10, TimeUnit.SECONDS);
+            advanceClockFuture.get(10, TimeUnit.SECONDS);
+            shrinkIsrFuture.get(10, TimeUnit.SECONDS);
+            assertThat(replica.getIsr()).containsExactly(1);
+        } finally {
+            executor.shutdownNow();
+        }
+    }
+
     @Test
     void testDoNotShrinkIsrAfterRepeatedNotifyLeaderAndIsr() throws Exception {
         TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
@@ -243,6 +324,17 @@ public class AdjustIsrTest extends ReplicaTestBase {
                 result -> {});
     }
 
+    private static void waitUntilQueued(
+            ReentrantReadWriteLock lock, AtomicReference<Thread> threadReference) {
+        retry(
+                Duration.ofSeconds(10),
+                () -> {
+                    Thread thread = threadReference.get();
+                    assertThat(thread).isNotNull();
+                    assertThat(lock.hasQueuedThread(thread)).isTrue();
+                });
+    }
+
     private void notifyLeaderAndIsr(
             Replica replica, List<Integer> replicas, int leaderEpoch, int bucketEpoch) {
         makeLeaderAndFollower(