You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2026/08/12 12:43:30 UTC
(fluss) branch main updated: [server] Fix ISR shrink race during leader transition (#3920)
This is an automated email from the ASF dual-hosted git repository.
wuchong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
The following commit(s) were added to refs/heads/main by this push:
new b2c511c18 [server] Fix ISR shrink race during leader transition (#3920)
b2c511c18 is described below
commit b2c511c18b5e8960ec6ee884fa46b70cc3bbd6dc
Author: hanliu0830 <ha...@gmail.com>
AuthorDate: Wed Aug 12 20:43:26 2026 +0800
[server] Fix ISR shrink race during leader transition (#3920)
---
.../coordinator/CoordinatorEventProcessor.java | 24 +++++
.../org/apache/fluss/server/replica/Replica.java | 49 +++++-----
.../coordinator/CoordinatorEventProcessorTest.java | 105 +++++++++++++++++++++
.../apache/fluss/server/replica/AdjustIsrTest.java | 92 ++++++++++++++++++
4 files changed, 247 insertions(+), 23 deletions(-)
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index e879a1586..96220fee1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -2035,6 +2035,30 @@ public class CoordinatorEventProcessor implements EventProcessor {
throw new InvalidUpdateVersionException(
"The request bucket epoch in adjust isr request is lower than current bucket epoch in coordinator.");
} else {
+ if (newLeaderAndIsr.leader() != currentLeaderAndIsr.leader()) {
+ String errorMsg =
+ String.format(
+ "Rejecting adjustIsr request for table bucket %s because request leader %s "
+ + "does not match current leader %s",
+ tableBucket,
+ newLeaderAndIsr.leader(),
+ currentLeaderAndIsr.leader());
+ LOG.error(errorMsg);
+ throw new FencedLeaderEpochException(errorMsg);
+ }
+
+ if (!newLeaderAndIsr.isr().contains(currentLeaderAndIsr.leader())) {
+ String errorMsg =
+ String.format(
+ "Rejecting adjustIsr request for table bucket %s because leader %s "
+ + "is not in the new ISR %s",
+ tableBucket,
+ currentLeaderAndIsr.leader(),
+ newLeaderAndIsr.isr());
+ LOG.error(errorMsg);
+ throw new IneligibleReplicaException(errorMsg);
+ }
+
// Check if the new ISR are all ineligible replicas (doesn't contain any shutting
// down tabletServers).
Set<Integer> ineligibleReplicas = new HashSet<>(newLeaderAndIsr.isr());
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index 5192cdeb0..784c4ad04 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -135,7 +135,6 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.IntSupplier;
import java.util.function.Supplier;
@@ -191,7 +190,7 @@ public final class Replica {
private final LogFormat logFormat;
private final ArrowCompressionInfo arrowCompressionInfo;
private final AtomicReference<Integer> leaderReplicaIdOpt = new AtomicReference<>();
- private final ReadWriteLock leaderIsrUpdateLock = new ReentrantReadWriteLock();
+ private final ReentrantReadWriteLock leaderIsrUpdateLock = new ReentrantReadWriteLock();
private final Clock clock;
private final RemoteLogManager remoteLogManager;
@@ -1865,25 +1864,25 @@ public final class Replica {
* <p>This function can be triggered when a replica's LEO has incremented.
*/
private void maybeExpandISr(FollowerReplica followerReplica) {
- IsrState currentIsrState = isrState;
boolean needsIsrUpdate =
- !currentIsrState.isInflight()
- && inReadLock(leaderIsrUpdateLock, () -> needsExpandIsr(followerReplica));
+ inReadLock(
+ leaderIsrUpdateLock,
+ () -> !isrState.isInflight() && needsExpandIsr(followerReplica));
if (needsIsrUpdate) {
Optional<IsrState.PendingExpandIsrState> adjustIsrUpdateOpt =
inWriteLock(
leaderIsrUpdateLock,
() -> {
+ IsrState currentIsrState = isrState;
// check if this replica needs to be added to the ISR.
- if (currentIsrState instanceof IsrState.CommittedIsrState) {
- if (needsExpandIsr(followerReplica)) {
- return Optional.of(
- prepareIsrExpand(
- (IsrState.CommittedIsrState)
- currentIsrState,
- followerReplica.getFollowerId()));
- }
+ if (isLeader()
+ && currentIsrState instanceof IsrState.CommittedIsrState
+ && needsExpandIsr(followerReplica)) {
+ return Optional.of(
+ prepareIsrExpand(
+ (IsrState.CommittedIsrState) currentIsrState,
+ followerReplica.getFollowerId()));
}
return Optional.empty();
@@ -1897,28 +1896,27 @@ public final class Replica {
}
void maybeShrinkIsr() {
- IsrState currentIstState = isrState;
boolean needsIsrUpdate =
- !currentIstState.isInflight()
- && inReadLock(leaderIsrUpdateLock, this::needsShrinkIsr);
+ inReadLock(leaderIsrUpdateLock, () -> !isrState.isInflight() && needsShrinkIsr());
if (needsIsrUpdate) {
Optional<IsrState.PendingShrinkIsrState> adjustIsrUpdateOpt =
inWriteLock(
leaderIsrUpdateLock,
() -> {
- if (isLeader()) {
+ IsrState currentIsrState = isrState;
+ if (isLeader()
+ && currentIsrState instanceof IsrState.CommittedIsrState) {
List<Integer> outOfSyncFollowerReplicas =
getOutOfSyncFollowerReplicas(replicaMaxLagTime);
- if (currentIstState instanceof IsrState.CommittedIsrState
- && !outOfSyncFollowerReplicas.isEmpty()) {
+ if (!outOfSyncFollowerReplicas.isEmpty()) {
List<Integer> newIsr =
- new ArrayList<>(currentIstState.isr());
+ new ArrayList<>(currentIsrState.isr());
newIsr.removeAll(outOfSyncFollowerReplicas);
LOG.info(
"Shrink ISR From {} to {} for bucket {}. Leader: (high watermark: {}, "
+ "end offset: {}, out of sync replicas: {})",
- currentIstState.isr(),
+ currentIsrState.isr(),
newIsr,
tableBucket,
logTablet.getHighWatermark(),
@@ -1927,7 +1925,7 @@ public final class Replica {
return Optional.of(
prepareIsrShrink(
(IsrState.CommittedIsrState)
- currentIstState,
+ currentIsrState,
newIsr,
outOfSyncFollowerReplicas));
}
@@ -1950,7 +1948,7 @@ public final class Replica {
// reflect the updated ISR even if there is a delay before we receive the confirmation.
// Alternatively, if the update fails, no harm is done since the expanded ISR puts
// a stricter requirement for advancement of the HW.
- List<Integer> isrToSend = new ArrayList<>(isrState.isr());
+ List<Integer> isrToSend = new ArrayList<>(currentState.isr());
isrToSend.add(newInSyncReplicaId);
// TODO add server epoch to isr.
@@ -2365,6 +2363,11 @@ public final class Replica {
return bucketEpoch;
}
+ @VisibleForTesting
+ ReentrantReadWriteLock getLeaderIsrUpdateLock() {
+ return leaderIsrUpdateLock;
+ }
+
@VisibleForTesting
public List<Integer> getIsr() {
return isrState.isr();
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
index cee0010fc..0bbb40b5c 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
@@ -1273,6 +1273,70 @@ class CoordinatorEventProcessorTest {
assertThat(resultForBucketMap.values()).allMatch(AdjustIsrResultForBucket::succeeded);
}
+ @Test
+ void testProcessAdjustIsrRejectsRequestFromNonLeader() throws Exception {
+ Map.Entry<TableBucket, LeaderAndIsr> entry =
+ createTableAndGetLeaderAndIsr("adjust_isr_from_non_leader");
+ TableBucket tableBucket = entry.getKey();
+ LeaderAndIsr currentLeaderAndIsr = entry.getValue();
+ int nonLeader =
+ currentLeaderAndIsr.isr().stream()
+ .filter(replica -> replica != currentLeaderAndIsr.leader())
+ .findFirst()
+ .get();
+ LeaderAndIsr invalidLeaderAndIsr =
+ new LeaderAndIsr(
+ nonLeader,
+ currentLeaderAndIsr.leaderEpoch(),
+ currentLeaderAndIsr.isr(),
+ currentLeaderAndIsr.standbyReplicas(),
+ currentLeaderAndIsr.coordinatorEpoch(),
+ currentLeaderAndIsr.bucketEpoch());
+
+ AdjustIsrResultForBucket invalidResult = submitAdjustIsr(tableBucket, invalidLeaderAndIsr);
+
+ assertThat(invalidResult.getError().error())
+ .isEqualTo(Errors.FENCED_LEADER_EPOCH_EXCEPTION);
+ assertThat(invalidResult.getErrorMessage())
+ .contains(
+ String.format(
+ "request leader %s does not match current leader %s",
+ nonLeader, currentLeaderAndIsr.leader()));
+ Optional<LeaderAndIsr> storedLeaderAndIsr =
+ fromCtx(ctx -> ctx.getBucketLeaderAndIsr(tableBucket));
+ assertThat(storedLeaderAndIsr).contains(currentLeaderAndIsr);
+ }
+
+ @Test
+ void testProcessAdjustIsrRejectsIsrWithoutCurrentLeader() throws Exception {
+ Map.Entry<TableBucket, LeaderAndIsr> entry =
+ createTableAndGetLeaderAndIsr("adjust_isr_without_current_leader");
+ TableBucket tableBucket = entry.getKey();
+ LeaderAndIsr currentLeaderAndIsr = entry.getValue();
+ List<Integer> isrWithoutLeader =
+ currentLeaderAndIsr.isr().stream()
+ .filter(replica -> replica != currentLeaderAndIsr.leader())
+ .collect(Collectors.toList());
+ LeaderAndIsr invalidLeaderAndIsr =
+ new LeaderAndIsr(
+ currentLeaderAndIsr.leader(),
+ currentLeaderAndIsr.leaderEpoch(),
+ isrWithoutLeader,
+ currentLeaderAndIsr.standbyReplicas(),
+ currentLeaderAndIsr.coordinatorEpoch(),
+ currentLeaderAndIsr.bucketEpoch());
+
+ AdjustIsrResultForBucket invalidResult = submitAdjustIsr(tableBucket, invalidLeaderAndIsr);
+ assertThat(invalidResult.getError().error()).isEqualTo(Errors.INELIGIBLE_REPLICA_EXCEPTION);
+ assertThat(invalidResult.getErrorMessage())
+ .contains(
+ String.format(
+ "leader %s is not in the new ISR", currentLeaderAndIsr.leader()));
+ Optional<LeaderAndIsr> storedLeaderAndIsr =
+ fromCtx(ctx -> ctx.getBucketLeaderAndIsr(tableBucket));
+ assertThat(storedLeaderAndIsr).contains(currentLeaderAndIsr);
+ }
+
@Test
void testDiskWriteLockedNotifyLeaderResponseMarksReplicaOffline() throws Exception {
initCoordinatorChannel();
@@ -2497,6 +2561,47 @@ class CoordinatorEventProcessorTest {
return event.getResultFuture().get(30, TimeUnit.SECONDS);
}
+ private Map.Entry<TableBucket, LeaderAndIsr> createTableAndGetLeaderAndIsr(String tableName)
+ throws Exception {
+ initCoordinatorChannel();
+ TableAssignment tableAssignment =
+ generateAssignment(
+ N_BUCKETS,
+ REPLICATION_FACTOR,
+ new TabletServerInfo[] {
+ new TabletServerInfo(0, "rack0"),
+ new TabletServerInfo(1, "rack1"),
+ new TabletServerInfo(2, "rack2")
+ });
+ long tableId =
+ metadataManager.createTable(
+ TablePath.of(defaultDatabase, tableName),
+ remoteDataDir,
+ TEST_TABLE,
+ tableAssignment,
+ false);
+ verifyTableCreated(tableId, tableAssignment, N_BUCKETS, REPLICATION_FACTOR);
+
+ Map<TableBucket, LeaderAndIsr> bucketLeaderAndIsrMap =
+ new HashMap<>(
+ waitValue(
+ () -> fromCtx(ctx -> Optional.of(ctx.bucketLeaderAndIsr())),
+ Duration.ofMinutes(1),
+ "leader not elected"));
+ return bucketLeaderAndIsrMap.entrySet().iterator().next();
+ }
+
+ private AdjustIsrResultForBucket submitAdjustIsr(
+ TableBucket tableBucket, LeaderAndIsr newLeaderAndIsr) throws Exception {
+ CompletableFuture<AdjustIsrResponse> response = new CompletableFuture<>();
+ eventProcessor
+ .getCoordinatorEventManager()
+ .put(
+ new AdjustIsrReceivedEvent(
+ Collections.singletonMap(tableBucket, newLeaderAndIsr), response));
+ return getAdjustIsrResponseData(response.get()).get(tableBucket);
+ }
+
private long createTable(TablePath tablePath, TabletServerInfo[] servers) {
TableAssignment tableAssignment =
generateAssignment(N_BUCKETS, REPLICATION_FACTOR, servers);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java
index 7a5459c3f..77e425bbc 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrTest.java
@@ -35,9 +35,14 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.fluss.record.TestData.DATA1;
+import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
@@ -126,6 +131,82 @@ public class AdjustIsrTest extends ReplicaTestBase {
assertThat(replica.getIsr()).containsExactlyInAnyOrder(1);
}
+ @Test
+ void testShrinkIsrUsesLatestStateAfterLeaderChange() throws Exception {
+ TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
+ Replica replica = makeLogReplica(DATA1_PHYSICAL_TABLE_PATH, tb);
+ NotifyLeaderAndIsrData leaderData =
+ new NotifyLeaderAndIsrData(
+ DATA1_PHYSICAL_TABLE_PATH,
+ tb,
+ Arrays.asList(1, 2),
+ new LeaderAndIsr(1, 0, Arrays.asList(1, 2), Collections.emptyList(), 0, 0));
+
+ ReentrantReadWriteLock leaderIsrUpdateLock = replica.getLeaderIsrUpdateLock();
+
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ try {
+ AtomicReference<Thread> makeLeaderThread = new AtomicReference<>();
+ AtomicReference<Thread> advanceClockThread = new AtomicReference<>();
+ AtomicReference<Thread> shrinkIsrThread = new AtomicReference<>();
+
+ leaderIsrUpdateLock.writeLock().lock();
+ CompletableFuture<Void> makeLeaderFuture;
+ CompletableFuture<Void> advanceClockFuture;
+ CompletableFuture<Void> shrinkIsrFuture;
+ try {
+ makeLeaderFuture =
+ CompletableFuture.runAsync(
+ () -> {
+ makeLeaderThread.set(Thread.currentThread());
+ try {
+ replica.makeLeader(leaderData);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ },
+ executor);
+ waitUntilQueued(leaderIsrUpdateLock, makeLeaderThread);
+
+ advanceClockFuture =
+ CompletableFuture.runAsync(
+ () -> {
+ advanceClockThread.set(Thread.currentThread());
+ leaderIsrUpdateLock.writeLock().lock();
+ try {
+ manualClock.advanceTime(
+ conf.get(ConfigOptions.LOG_REPLICA_MAX_LAG_TIME)
+ .toMillis()
+ + 1,
+ TimeUnit.MILLISECONDS);
+ } finally {
+ leaderIsrUpdateLock.writeLock().unlock();
+ }
+ },
+ executor);
+ waitUntilQueued(leaderIsrUpdateLock, advanceClockThread);
+
+ shrinkIsrFuture =
+ CompletableFuture.runAsync(
+ () -> {
+ shrinkIsrThread.set(Thread.currentThread());
+ replica.maybeShrinkIsr();
+ },
+ executor);
+ waitUntilQueued(leaderIsrUpdateLock, shrinkIsrThread);
+ } finally {
+ leaderIsrUpdateLock.writeLock().unlock();
+ }
+
+ makeLeaderFuture.get(10, TimeUnit.SECONDS);
+ advanceClockFuture.get(10, TimeUnit.SECONDS);
+ shrinkIsrFuture.get(10, TimeUnit.SECONDS);
+ assertThat(replica.getIsr()).containsExactly(1);
+ } finally {
+ executor.shutdownNow();
+ }
+ }
+
@Test
void testDoNotShrinkIsrAfterRepeatedNotifyLeaderAndIsr() throws Exception {
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
@@ -243,6 +324,17 @@ public class AdjustIsrTest extends ReplicaTestBase {
result -> {});
}
+ private static void waitUntilQueued(
+ ReentrantReadWriteLock lock, AtomicReference<Thread> threadReference) {
+ retry(
+ Duration.ofSeconds(10),
+ () -> {
+ Thread thread = threadReference.get();
+ assertThat(thread).isNotNull();
+ assertThat(lock.hasQueuedThread(thread)).isTrue();
+ });
+ }
+
private void notifyLeaderAndIsr(
Replica replica, List<Integer> replicas, int leaderEpoch, int bucketEpoch) {
makeLeaderAndFollower(