You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2026/04/09 11:37:14 UTC
(fluss) 02/02: [server] Update leader election process and improve coordinator epoch handling
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
commit 2c031c480aed23bbee27401e18185de6e9d97071
Author: Jark Wu <ja...@apache.org>
AuthorDate: Thu Apr 9 15:05:09 2026 +0800
[server] Update leader election process and improve coordinator epoch handling
---
.../server/coordinator/CoordinatorContext.java | 29 ++++++++++------
.../coordinator/CoordinatorEventProcessor.java | 16 ++++-----
.../coordinator/CoordinatorLeaderElection.java | 14 +++-----
.../server/coordinator/CoordinatorServer.java | 32 ++++--------------
.../statemachine/ReplicaStateMachine.java | 2 +-
.../statemachine/TableBucketStateMachine.java | 11 +++---
.../fluss/server/replica/ReplicaManager.java | 17 ++++++----
.../java/org/apache/fluss/server/zk/ZkEpoch.java | 7 ++++
.../apache/fluss/server/zk/ZooKeeperClient.java | 37 ++++++++++++++------
.../server/coordinator/CoordinatorContextTest.java | 3 +-
.../coordinator/CoordinatorEventProcessorTest.java | 3 +-
.../CoordinatorHighAvailabilityITCase.java | 33 ++++++++++++------
.../fluss/server/coordinator/TableManagerTest.java | 7 ++--
.../server/coordinator/TestCoordinatorContext.java | 39 ----------------------
.../rebalance/RebalanceManagerTest.java | 3 +-
.../statemachine/ReplicaStateMachineTest.java | 18 +++++-----
.../statemachine/TableBucketStateMachineTest.java | 9 ++---
.../server/testutils/FlussClusterExtension.java | 7 +++-
18 files changed, 132 insertions(+), 155 deletions(-)
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java
index 138ddb175..92dfc5ee5 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java
@@ -28,6 +28,7 @@ import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.server.coordinator.statemachine.BucketState;
import org.apache.fluss.server.coordinator.statemachine.ReplicaState;
import org.apache.fluss.server.metadata.ServerInfo;
+import org.apache.fluss.server.zk.ZkEpoch;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
import org.apache.fluss.utils.types.Tuple2;
@@ -108,25 +109,33 @@ public class CoordinatorContext {
/** A mapping from tabletServers to server tag. */
private final Map<Integer, ServerTag> serverTags = new HashMap<>();
+ /**
+ * The epoch of the coordinator, which will be incremented whenever the coordinator is elected
+ * or re-elected.
+ */
+ private final int coordinatorEpoch;
+
+ /**
+ * The coordinator epoch zk version is the zk version when the coordinator epoch is updated in
+ * Zookeeper.
+ */
+ private final int coordinatorEpochZkVersion;
+
private ServerInfo coordinatorServerInfo = null;
- private int coordinatorEpoch = INITIAL_COORDINATOR_EPOCH;
- private int coordinatorEpochZkVersion = INITIAL_COORDINATOR_EPOCH_ZK_VERSION;
- public CoordinatorContext() {}
+ public CoordinatorContext(ZkEpoch zkEpoch) {
+ this.coordinatorEpoch = zkEpoch.getCoordinatorEpoch();
+ this.coordinatorEpochZkVersion = zkEpoch.getCoordinatorEpochZkVersion();
+ }
public int getCoordinatorEpoch() {
return coordinatorEpoch;
}
- public int getCoordinatorEpochZkVersion() {
+ public int getCoordinatorZkVersion() {
return coordinatorEpochZkVersion;
}
- public void setCoordinatorEpochAndZkVersion(int newEpoch, int newZkVersion) {
- this.coordinatorEpoch = newEpoch;
- this.coordinatorEpochZkVersion = newZkVersion;
- }
-
public Set<String> getLiveCoordinatorServers() {
return liveCoordinatorServers;
}
@@ -719,8 +728,6 @@ public class CoordinatorContext {
public void resetContext() {
tablesToBeDeleted.clear();
- coordinatorEpoch = INITIAL_COORDINATOR_EPOCH;
- coordinatorEpochZkVersion = INITIAL_COORDINATOR_EPOCH_ZK_VERSION;
clearTablesState();
liveTabletServers.clear();
liveCoordinatorServers.clear();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index 423d27622..7ae553caa 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -108,7 +108,6 @@ import org.apache.fluss.server.metadata.CoordinatorMetadataCache;
import org.apache.fluss.server.metadata.ServerInfo;
import org.apache.fluss.server.metrics.group.CoordinatorMetricGroup;
import org.apache.fluss.server.utils.ServerRpcMessageUtils;
-import org.apache.fluss.server.zk.ZkEpoch;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.data.BucketAssignment;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
@@ -189,7 +188,6 @@ public class CoordinatorEventProcessor implements EventProcessor {
public CoordinatorEventProcessor(
ZooKeeperClient zooKeeperClient,
- ZkEpoch zkEpoch,
CoordinatorMetadataCache serverMetadataCache,
CoordinatorChannelManager coordinatorChannelManager,
CoordinatorContext coordinatorContext,
@@ -255,9 +253,6 @@ public class CoordinatorEventProcessor implements EventProcessor {
this.ioExecutor = ioExecutor;
this.lakeTableHelper =
new LakeTableHelper(zooKeeperClient, conf.getString(ConfigOptions.REMOTE_DATA_DIR));
-
- this.coordinatorContext.setCoordinatorEpochAndZkVersion(
- zkEpoch.getCoordinatorEpoch(), zkEpoch.getCoordinatorEpochZkVersion());
}
public CoordinatorEventManager getCoordinatorEventManager() {
@@ -314,6 +309,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
coordinatorEventManager.close();
rebalanceManager.close();
onShutdown();
+ coordinatorContext.resetContext();
}
private ServerInfo getCoordinatorServerInfo() {
@@ -1626,7 +1622,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
zooKeeperClient.updateTableAssignment(
tableId,
new TableAssignment(newTableAssignment),
- coordinatorContext.getCoordinatorEpochZkVersion());
+ coordinatorContext.getCoordinatorZkVersion());
} else {
Map<Integer, List<Integer>> partitionAssignment =
coordinatorContext.getPartitionAssignment(
@@ -1639,7 +1635,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
zooKeeperClient.updatePartitionAssignment(
partitionId,
new PartitionAssignment(tableId, newPartitionAssignment),
- coordinatorContext.getCoordinatorEpochZkVersion());
+ coordinatorContext.getCoordinatorZkVersion());
}
}
@@ -1686,7 +1682,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
try {
zooKeeperClient.batchUpdateLeaderAndIsr(
- newLeaderAndIsrList, coordinatorContext.getCoordinatorEpochZkVersion());
+ newLeaderAndIsrList, coordinatorContext.getCoordinatorZkVersion());
newLeaderAndIsrList.forEach(
(tableBucket, newLeaderAndIsr) ->
result.add(new AdjustIsrResultForBucket(tableBucket, newLeaderAndIsr)));
@@ -1700,7 +1696,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
zooKeeperClient.updateLeaderAndIsr(
tableBucket,
newLeaderAndIsr,
- coordinatorContext.getCoordinatorEpochZkVersion());
+ coordinatorContext.getCoordinatorZkVersion());
} catch (Exception e) {
LOG.error("Error when register leader and isr.", e);
result.add(
@@ -2227,7 +2223,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
coordinatorContext.putBucketLeaderAndIsr(tableBucket, newLeaderAndIsr);
zooKeeperClient.updateLeaderAndIsr(
- tableBucket, newLeaderAndIsr, coordinatorContext.getCoordinatorEpochZkVersion());
+ tableBucket, newLeaderAndIsr, coordinatorContext.getCoordinatorZkVersion());
coordinatorRequestBatch.newBatch();
coordinatorRequestBatch.addNotifyLeaderRequestForTabletServers(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java
index d67f77fdf..f368e279e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java
@@ -52,8 +52,6 @@ public class CoordinatorLeaderElection implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(CoordinatorLeaderElection.class);
private final String serverId;
- private final ZooKeeperClient zkClient;
- private final CoordinatorContext coordinatorContext;
private final LeaderLatch leaderLatch;
private final AtomicBoolean isLeader = new AtomicBoolean(false);
// Cached thread pool to run leader init/cleanup callbacks outside Curator's EventThread.
@@ -66,11 +64,8 @@ public class CoordinatorLeaderElection implements AutoCloseable {
private final AtomicReference<CompletableFuture<Void>> pendingCleanup =
new AtomicReference<>(CompletableFuture.completedFuture(null));
- public CoordinatorLeaderElection(
- ZooKeeperClient zkClient, String serverId, CoordinatorContext coordinatorContext) {
+ public CoordinatorLeaderElection(ZooKeeperClient zkClient, String serverId) {
this.serverId = serverId;
- this.zkClient = zkClient;
- this.coordinatorContext = coordinatorContext;
this.leaderLatch =
new LeaderLatch(
zkClient.getCuratorClient(),
@@ -113,10 +108,6 @@ public class CoordinatorLeaderElection implements AutoCloseable {
// Run init on a separate thread to avoid deadlock with
// Curator's EventThread when performing ZK operations.
- // Set leader flag before init completes, so when zk found this leader, the
- // leader can accept requests
- isLeader.set(true);
-
leaderCallbackExecutor.execute(
() -> {
// Wait for any pending cleanup to finish first.
@@ -134,6 +125,9 @@ public class CoordinatorLeaderElection implements AutoCloseable {
}
try {
initLeaderServices.run();
+ // Set leader flag after init completes, so when zk found
+ // this leader, the leader can accept requests
+ isLeader.set(true);
} catch (CoordinatorEpochFencedException e) {
LOG.warn(
"Coordinator server {} has been fenced and not become leader successfully.",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java
index a54fc2d58..f44bbd7a8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java
@@ -141,9 +141,6 @@ public class CoordinatorServer extends ServerBase {
@Nullable
private Authorizer authorizer;
- @GuardedBy("lock")
- private CoordinatorContext coordinatorContext;
-
@GuardedBy("lock")
private DynamicConfigManager dynamicConfigManager;
@@ -221,11 +218,9 @@ public class CoordinatorServer extends ServerBase {
serverId);
this.zkClient = ZooKeeperUtils.startZookeeperClient(conf, this);
- this.coordinatorContext = new CoordinatorContext();
// CoordinatorLeaderElection must be created after zkClient is initialized.
- this.coordinatorLeaderElection =
- new CoordinatorLeaderElection(zkClient, serverId, coordinatorContext);
+ this.coordinatorLeaderElection = new CoordinatorLeaderElection(zkClient, serverId);
this.lakeCatalogDynamicLoader = new LakeCatalogDynamicLoader(conf, pluginManager, true);
this.dynamicConfigManager = new DynamicConfigManager(zkClient, conf, true);
@@ -298,8 +293,8 @@ public class CoordinatorServer extends ServerBase {
protected void initCoordinatorLeader() throws Exception {
// to avoid split-brain
ZkEpoch zkEpoch = zkClient.fenceBecomeCoordinatorLeader(serverId);
-
registerCoordinatorLeader();
+
synchronized (lock) {
this.clientMetricGroup = new ClientMetricGroup(metricRegistry, SERVER_NAME);
this.rpcClient = RpcClient.create(conf, clientMetricGroup);
@@ -311,14 +306,13 @@ public class CoordinatorServer extends ServerBase {
autoPartitionManager.start();
// start coordinator event processor after we register coordinator leader to zk
- // so that the event processor can get the coordinator leader node from zk during start
- // up.
- // in HA for coordinator server, the processor also need to know the leader node during
- // start up
+ // so that the event processor can get the coordinator leader node from zk during
+ // start up. In HA for coordinator server, the processor also need to know the leader
+ // node during start up
+ CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
this.coordinatorEventProcessor =
new CoordinatorEventProcessor(
zkClient,
- zkEpoch,
metadataCache,
coordinatorChannelManager,
coordinatorContext,
@@ -402,11 +396,6 @@ public class CoordinatorServer extends ServerBase {
LOG.warn("Failed to close client metric group", t);
}
- // Reset coordinator context for next election
- if (coordinatorContext != null) {
- coordinatorContext.resetContext();
- }
-
LOG.info("Coordinator leader services cleaned up successfully.");
}
}
@@ -601,15 +590,6 @@ public class CoordinatorServer extends ServerBase {
exception = ExceptionUtils.firstOrSuppressed(t, exception);
}
- try {
- if (coordinatorContext != null) {
- // then reset coordinatorContext
- coordinatorContext.resetContext();
- }
- } catch (Throwable t) {
- exception = ExceptionUtils.firstOrSuppressed(t, exception);
- }
-
try {
if (lakeTableTieringManager != null) {
lakeTableTieringManager.close();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java
index 2416b163b..63f5dd965 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java
@@ -488,7 +488,7 @@ public class ReplicaStateMachine {
}
try {
zooKeeperClient.batchUpdateLeaderAndIsr(
- toUpdateLeaderAndIsrList, coordinatorContext.getCoordinatorEpochZkVersion());
+ toUpdateLeaderAndIsrList, coordinatorContext.getCoordinatorZkVersion());
toUpdateLeaderAndIsrList.forEach(coordinatorContext::putBucketLeaderAndIsr);
return adjustedLeaderAndIsr;
} catch (Exception e) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java
index b01e65c92..300e98f08 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java
@@ -312,9 +312,7 @@ public class TableBucketStateMachine {
LeaderAndIsr leaderAndIsr = electionResult.leaderAndIsr;
try {
zooKeeperClient.registerLeaderAndIsr(
- tableBucket,
- leaderAndIsr,
- coordinatorContext.getCoordinatorEpochZkVersion());
+ tableBucket, leaderAndIsr, coordinatorContext.getCoordinatorZkVersion());
} catch (Exception e) {
LOG.error(
"Fail to create state node for table bucket {} in zookeeper.",
@@ -382,8 +380,7 @@ public class TableBucketStateMachine {
if (!tableBucketLeadAndIsrInfos.isEmpty()) {
try {
zooKeeperClient.batchRegisterLeaderAndIsrForTablePartition(
- tableBucketLeadAndIsrInfos,
- coordinatorContext.getCoordinatorEpochZkVersion());
+ tableBucketLeadAndIsrInfos, coordinatorContext.getCoordinatorZkVersion());
registerSuccessList.addAll(tableBucketLeadAndIsrInfos);
} catch (Exception e) {
LOG.error(
@@ -461,7 +458,7 @@ public class TableBucketStateMachine {
zooKeeperClient.registerLeaderAndIsr(
info.getTableBucket(),
info.getLeaderAndIsr(),
- coordinatorContext.getCoordinatorEpochZkVersion());
+ coordinatorContext.getCoordinatorZkVersion());
registerSuccessList.add(info);
} catch (Exception e) {
LOG.error(
@@ -506,7 +503,7 @@ public class TableBucketStateMachine {
zooKeeperClient.updateLeaderAndIsr(
tableBucket,
electionResult.leaderAndIsr,
- coordinatorContext.getCoordinatorEpochZkVersion());
+ coordinatorContext.getCoordinatorZkVersion());
} catch (Exception e) {
LOG.error(
"Fail to update bucket LeaderAndIsr for table bucket {}.",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index 6960b5191..19ab3f936 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -336,6 +336,11 @@ public class ReplicaManager implements ServerReconfigurable {
return minInSyncReplicas;
}
+ @VisibleForTesting
+ public int getCoordinatorEpoch() {
+ return coordinatorEpoch;
+ }
+
// ============ ServerReconfigurable Implementation ============
@Override
@@ -1879,9 +1884,14 @@ public class ReplicaManager implements ServerReconfigurable {
requestCoordinatorEpoch, requestName, this.coordinatorEpoch);
LOG.warn("Ignore the {} request because {}", requestName, errorMessage);
throw new InvalidCoordinatorException(errorMessage);
- } else {
+ } else if (requestCoordinatorEpoch > this.coordinatorEpoch) {
+ LOG.info(
+ "Update coordinator epoch from {} to {} for coordinator leader switch.",
+ this.coordinatorEpoch,
+ requestCoordinatorEpoch);
this.coordinatorEpoch = requestCoordinatorEpoch;
}
+ // ignore equal case
}
private void dropEmptyTableOrPartitionDir(Path dir, long id, String dirType) {
@@ -1991,11 +2001,6 @@ public class ReplicaManager implements ServerReconfigurable {
return serverMetricGroup;
}
- @VisibleForTesting
- public void resetCoordinatorEpoch() {
- this.coordinatorEpoch = CoordinatorContext.INITIAL_COORDINATOR_EPOCH;
- }
-
/**
* Interface to represent the state of hosted {@link Replica}. We create a concrete (active)
* {@link Replica} instance when the TabletServer receives a createLogLeader request or
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java
index f3d401a25..3f3984373 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java
@@ -18,8 +18,15 @@
package org.apache.fluss.server.zk;
+import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH;
+import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH_ZK_VERSION;
+
/** Class for coordinator epoch and coordinator epoch zk version. */
public class ZkEpoch {
+
+ public static final ZkEpoch INITIAL_EPOCH =
+ new ZkEpoch(INITIAL_COORDINATOR_EPOCH, INITIAL_COORDINATOR_EPOCH_ZK_VERSION);
+
private final int coordinatorEpoch;
private final int coordinatorEpochZkVersion;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
index 3c8b58fa2..cb34a2f2e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
@@ -139,24 +139,39 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull;
*
* <p>In some method, 'expectedZkVersion' is used to execute an epoch Zookeeper version check.
* Conditions requiring epoch checks (all must be met):
- * ┌─────────────────────────────────────────────────────────┐ │ 1. Invoked by the Coordinator (not
- * the TabletServer) │ │ 2. Operates on persistent nodes (not ephemeral) │ │ 3. Constitutes a
- * "control plane" operation: │ │ partition assignment or LeaderAndIsr election │ │ 4. Concurrent
- * access to the same path by old and new │ │ leaders during leader failover │ │ 5. No other
- * mechanisms (optimistic locking, │ │ idempotency, or reloading) provide fallback │
+ *
+ * <pre>
+ * ┌─────────────────────────────────────────────────────────┐
+ * │ 1. Invoked by the Coordinator (not the TabletServer) │
+ * │ 2. Operates on persistent nodes (not ephemeral) │
+ * │ 3. Constitutes a "control plane" operation: │
+ * │ partition assignment or LeaderAndIsr election │
+ * │ 4. Concurrent access to the same path by old and new │
+ * │ leaders during leader failover │
+ * │ 5. No other mechanisms (optimistic locking, │
+ * │ idempotency, or reloading) provide fallback │
* └─────────────────────────────────────────────────────────┘
+ * </pre>
+ *
+ * <p>In practice, only two types of operations truly require this:
*
- * <p>In practice, only two types of operations truly require this: - CRUD for Table/Partition
- * Assignment (assignment decisions). - CRUD for LeaderAndIsr (leader election results).
+ * <ul>
+ * <li>CRUD for Table/Partition Assignment (assignment decisions).
+ * <li>CRUD for LeaderAndIsr (leader election results).
+ * </ul>
*
* <p>These operations are inevitably executed concurrently by the old and new coordinators during
* failover (as the new leader immediately reassigns partitions), and overwrites cannot be
* automatically recovered.
*
- * <p>All other operations do not require epoch checks because: - DDL operations are protected
- * against concurrency by client reconnection mechanisms. - TabletServer operations are unaffected
- * by coordinator failovers. - ACLs and Configs have their own version control or idempotency
- * guarantees. - Ephemeral nodes are managed via session lifecycle.
+ * <p>All other operations do not require epoch checks because:
+ *
+ * <ul>
+ * <li>DDL operations are protected against concurrency by client reconnection mechanisms.
+ * <li>TabletServer operations are unaffected by coordinator failovers.
+ * <li>ACLs and Configs have their own version control or idempotency guarantees.
+ * <li>Ephemeral nodes are managed via session lifecycle.
+ * </ul>
*/
@Internal
public class ZooKeeperClient implements AutoCloseable {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java
index 5c321dfff..2b788bdb4 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java
@@ -22,6 +22,7 @@ import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.server.zk.ZkEpoch;
import org.apache.fluss.types.DataTypes;
import org.junit.jupiter.api.Test;
@@ -37,7 +38,7 @@ class CoordinatorContextTest {
@Test
void testGetLakeTableCount() {
- CoordinatorContext context = new CoordinatorContext();
+ CoordinatorContext context = new CoordinatorContext(ZkEpoch.INITIAL_EPOCH);
// Initially, there should be no tables
assertThat(context.allTables()).isEmpty();
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
index cd036b364..63f0b5fd9 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
@@ -1061,10 +1061,9 @@ class CoordinatorEventProcessorTest {
private CoordinatorEventProcessor buildCoordinatorEventProcessor() {
return new CoordinatorEventProcessor(
zookeeperClient,
- zkEpoch,
serverMetadataCache,
testCoordinatorChannelManager,
- new TestCoordinatorContext(),
+ new CoordinatorContext(zkEpoch),
autoPartitionManager,
lakeTableTieringManager,
TestingMetricGroups.COORDINATOR_METRICS,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java
index e974ec2d9..68e46e359 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java
@@ -283,16 +283,14 @@ class CoordinatorHighAvailabilityITCase {
.isEqualTo(standby.getServerId());
int newCoordinatorEpoch = standby.getCoordinatorEventProcessor().getCoordinatorEpoch();
+ // wait the new coordinator notifies the tablet server about the new coordinator epoch
+ waitUntil(
+ () -> tabletServer.getReplicaManager().getCoordinatorEpoch() == newCoordinatorEpoch,
+ Duration.ofSeconds(30),
+ "Tablet server did not update to new coordinator epoch");
TabletServerGateway tsGateway = createGatewayForTabletServer(tabletServer);
- // Send request with new coordinator epoch first to ensure the tablet server
- // has updated its stored epoch to the latest value
- tsGateway
- .updateMetadata(
- new UpdateMetadataRequest().setCoordinatorEpoch(newCoordinatorEpoch))
- .get();
-
// Send request with old coordinator epoch — tablet server should reject it
assertThatThrownBy(
() ->
@@ -339,11 +337,11 @@ class CoordinatorHighAvailabilityITCase {
int newLeaderEpochZkVersion =
standby.getCoordinatorEventProcessor()
.getCoordinatorContext()
- .getCoordinatorEpochZkVersion();
+ .getCoordinatorZkVersion();
int oldLeaderEpochZkVersion =
leader.getCoordinatorEventProcessor()
.getCoordinatorContext()
- .getCoordinatorEpochZkVersion();
+ .getCoordinatorZkVersion();
assertThatThrownBy(
() ->
@@ -413,11 +411,23 @@ class CoordinatorHighAvailabilityITCase {
return configuration;
}
- private void waitUntilCoordinatorServerElected() {
+ private void waitUntilCoordinatorServerElected() throws Exception {
waitUntil(
() -> zookeeperClient.getCoordinatorLeaderAddress().isPresent(),
Duration.ofMinutes(1),
"Fail to wait coordinator server elected");
+ waitUntilCoordinatorLeaderReady();
+ }
+
+ private void waitUntilCoordinatorLeaderReady() throws Exception {
+ CoordinatorAddress currentLeaderAddress =
+ zookeeperClient.getCoordinatorLeaderAddress().get();
+ CoordinatorServer currentLeader = findServerById(currentLeaderAddress.getId());
+ // we need to wait some time for the coordinator leader start services after election
+ waitUntil(
+ () -> currentLeader.getCoordinatorService().isLeader(),
+ Duration.ofSeconds(30),
+ "Coordinator leader did not recognize itself as leader");
}
private CoordinatorServer findServerById(String serverId) {
@@ -493,7 +503,7 @@ class CoordinatorHighAvailabilityITCase {
}
/** Waits until a new coordinator leader is elected that is different from the given old one. */
- private void waitUntilNewLeaderElected(String oldLeaderId) {
+ private void waitUntilNewLeaderElected(String oldLeaderId) throws Exception {
waitUntil(
() -> {
try {
@@ -507,6 +517,7 @@ class CoordinatorHighAvailabilityITCase {
},
Duration.ofMinutes(1),
"Fail to wait for new coordinator leader to be elected");
+ waitUntilCoordinatorLeaderReady();
}
/** Verifies that the given server can process requests as a leader. */
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
index 2c30fc748..f5357c549 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
@@ -31,6 +31,7 @@ import org.apache.fluss.server.coordinator.statemachine.TableBucketStateMachine;
import org.apache.fluss.server.entity.DeleteReplicaResultForBucket;
import org.apache.fluss.server.metadata.ServerInfo;
import org.apache.fluss.server.zk.NOPErrorHandler;
+import org.apache.fluss.server.zk.ZkEpoch;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.ZooKeeperExtension;
import org.apache.fluss.server.zk.data.BucketAssignment;
@@ -78,6 +79,7 @@ class TableManagerTest {
new AllCallbackWrapper<>(new ZooKeeperExtension());
private static ZooKeeperClient zookeeperClient;
+ private static ZkEpoch zkEpoch;
private static ExecutorService ioExecutor;
private CoordinatorContext coordinatorContext;
@@ -86,11 +88,12 @@ class TableManagerTest {
private TestCoordinatorChannelManager testCoordinatorChannelManager;
@BeforeAll
- static void baseBeforeAll() {
+ static void baseBeforeAll() throws Exception {
zookeeperClient =
ZOO_KEEPER_EXTENSION_WRAPPER
.getCustomExtension()
.getZooKeeperClient(NOPErrorHandler.INSTANCE);
+ zkEpoch = zookeeperClient.fenceBecomeCoordinatorLeader("1");
ioExecutor = Executors.newFixedThreadPool(1);
}
@@ -113,7 +116,7 @@ class TableManagerTest {
private void initTableManager() {
testingEventManager = new TestingEventManager();
- coordinatorContext = new TestCoordinatorContext();
+ coordinatorContext = new CoordinatorContext(zkEpoch);
testCoordinatorChannelManager = new TestCoordinatorChannelManager();
Configuration conf = new Configuration();
conf.setString(ConfigOptions.REMOTE_DATA_DIR, "/tmp/fluss/remote-data");
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorContext.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorContext.java
deleted file mode 100644
index b931d564f..000000000
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorContext.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.fluss.server.coordinator;
-
-import org.apache.fluss.server.zk.data.ZkVersion;
-
-/** A coordinator context for test purpose which can set epoch manually. */
-public class TestCoordinatorContext extends CoordinatorContext {
- public TestCoordinatorContext() {
- // When create or modify ZooKeeper node, it should check ZooKeeper epoch node version
- // to ensure the coordinator is still holding the leadership. However, in the test
- // cases, we don't register epoch node, so we skip the check process by setting
- // "coordinatorEpochZkVersion" to ZkVersion.MATCH_ANY_VERSION
- super();
- this.setCoordinatorEpochAndZkVersion(
- INITIAL_COORDINATOR_EPOCH, ZkVersion.MATCH_ANY_VERSION.getVersion());
- }
-
- public TestCoordinatorContext(int coordinatorEpoch, int coordinatorEpochZkVersion) {
- super();
- this.setCoordinatorEpochAndZkVersion(coordinatorEpoch, coordinatorEpochZkVersion);
- }
-}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java
index 57ba9fff7..6ae36b8de 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java
@@ -137,10 +137,9 @@ public class RebalanceManagerTest {
private CoordinatorEventProcessor buildCoordinatorEventProcessor() {
return new CoordinatorEventProcessor(
zookeeperClient,
- zkEpoch,
serverMetadataCache,
testCoordinatorChannelManager,
- new CoordinatorContext(),
+ new CoordinatorContext(zkEpoch),
autoPartitionManager,
lakeTableTieringManager,
TestingMetricGroups.COORDINATOR_METRICS,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java
index 4c89b5722..f1ad2fe1c 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java
@@ -30,11 +30,11 @@ import org.apache.fluss.server.coordinator.CoordinatorContext;
import org.apache.fluss.server.coordinator.CoordinatorRequestBatch;
import org.apache.fluss.server.coordinator.CoordinatorTestUtils;
import org.apache.fluss.server.coordinator.TestCoordinatorChannelManager;
-import org.apache.fluss.server.coordinator.TestCoordinatorContext;
import org.apache.fluss.server.coordinator.event.DeleteReplicaResponseReceivedEvent;
import org.apache.fluss.server.entity.DeleteReplicaResultForBucket;
import org.apache.fluss.server.metadata.ServerInfo;
import org.apache.fluss.server.zk.NOPErrorHandler;
+import org.apache.fluss.server.zk.ZkEpoch;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.ZooKeeperExtension;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
@@ -71,18 +71,20 @@ class ReplicaStateMachineTest {
new AllCallbackWrapper<>(new ZooKeeperExtension());
private static ZooKeeperClient zookeeperClient;
+ private static ZkEpoch zkEpoch;
@BeforeAll
- static void baseBeforeAll() {
+ static void baseBeforeAll() throws Exception {
zookeeperClient =
ZOO_KEEPER_EXTENSION_WRAPPER
.getCustomExtension()
.getZooKeeperClient(NOPErrorHandler.INSTANCE);
+ zkEpoch = zookeeperClient.fenceBecomeCoordinatorLeader("1");
}
@Test
void testStartup() {
- CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+ CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
// init coordinator server context with a table assignment
TableBucket tableBucket = new TableBucket(1, 0);
@@ -107,7 +109,7 @@ class ReplicaStateMachineTest {
@Test
void testReplicaStateChange() {
- CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+ CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
ReplicaStateMachine replicaStateMachine = createReplicaStateMachine(coordinatorContext);
// test check valid replica state change
@@ -136,7 +138,7 @@ class ReplicaStateMachineTest {
@Test
void testDeleteReplicaStateChange() {
Map<TableBucketReplica, Boolean> isReplicaDeleteSuccess = new HashMap<>();
- CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+ CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
coordinatorContext.setLiveTabletServers(
CoordinatorTestUtils.createServers(Arrays.asList(0, 1)));
// use a context that will return a gateway that always get success ack
@@ -167,7 +169,7 @@ class ReplicaStateMachineTest {
}
// now, we change a context that some gateway will return exception
- coordinatorContext = new CoordinatorContext();
+ coordinatorContext = new CoordinatorContext(zkEpoch);
coordinatorContext.setLiveTabletServers(
CoordinatorTestUtils.createServers(Arrays.asList(0, 1)));
coordinatorContext.putBucketLeaderAndIsr(tableBucket1, new LeaderAndIsr(0, 0));
@@ -193,7 +195,7 @@ class ReplicaStateMachineTest {
@Test
void testOfflineReplicasShouldBeRemovedFromIsr() throws Exception {
- CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+ CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
coordinatorContext.setLiveTabletServers(createServers(new int[] {0, 1, 2}));
ReplicaStateMachine replicaStateMachine = createReplicaStateMachine(coordinatorContext);
@@ -233,7 +235,7 @@ class ReplicaStateMachineTest {
@Test
void testOfflineReplicaShouldBeRemovedFromIsr() throws Exception {
- CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+ CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
coordinatorContext.setLiveTabletServers(createServers(new int[] {0, 1, 2}));
ReplicaStateMachine replicaStateMachine = createReplicaStateMachine(coordinatorContext);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java
index d6697f8cb..2ea3718ff 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java
@@ -34,7 +34,6 @@ import org.apache.fluss.server.coordinator.LakeCatalogDynamicLoader;
import org.apache.fluss.server.coordinator.LakeTableTieringManager;
import org.apache.fluss.server.coordinator.MetadataManager;
import org.apache.fluss.server.coordinator.TestCoordinatorChannelManager;
-import org.apache.fluss.server.coordinator.TestCoordinatorContext;
import org.apache.fluss.server.coordinator.event.CoordinatorEventManager;
import org.apache.fluss.server.coordinator.lease.KvSnapshotLeaseManager;
import org.apache.fluss.server.coordinator.statemachine.ReplicaLeaderElection.ControlledShutdownLeaderElection;
@@ -57,7 +56,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
-import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
@@ -106,12 +104,12 @@ class TableBucketStateMachineTest {
}
@BeforeEach
- void beforeEach() throws IOException {
+ void beforeEach() {
Configuration conf = new Configuration();
conf.setString(ConfigOptions.COORDINATOR_HOST, "localhost");
String remoteDir = "/tmp/fluss/remote-data";
conf.setString(ConfigOptions.REMOTE_DATA_DIR, remoteDir);
- coordinatorContext = new TestCoordinatorContext();
+ coordinatorContext = new CoordinatorContext(zkEpoch);
testCoordinatorChannelManager = new TestCoordinatorChannelManager();
coordinatorRequestBatch =
new CoordinatorRequestBatch(
@@ -233,8 +231,6 @@ class TableBucketStateMachineTest {
coordinatorContext.putTablePath(tableId, fakeTablePath);
coordinatorContext.updateBucketReplicaAssignment(tableBucket, Arrays.asList(0, 1, 2));
coordinatorContext.putBucketState(tableBucket, NewBucket);
- coordinatorContext.setCoordinatorEpochAndZkVersion(
- 0, ZkVersion.MATCH_ANY_VERSION.getVersion());
// case1: init a new leader for NewBucket to OnlineBucket
tableBucketStateMachine.handleStateChange(Collections.singleton(tableBucket), OnlineBucket);
// non any alive servers, the state change fail
@@ -280,7 +276,6 @@ class TableBucketStateMachineTest {
CoordinatorEventProcessor coordinatorEventProcessor =
new CoordinatorEventProcessor(
zookeeperClient,
- zkEpoch,
serverMetadataCache,
new CoordinatorChannelManager(
RpcClient.create(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
index 9498419e0..74a041596 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
@@ -942,11 +942,16 @@ public final class FlussClusterExtension
return coordinatorServer;
}
- public void waitUntilCoordinatorServerElected() throws Exception {
+ private void waitUntilCoordinatorServerElected() {
waitUntil(
() -> zooKeeperClient.getCoordinatorLeaderAddress().isPresent(),
Duration.ofSeconds(10),
"Fail to wait coordinator server elected");
+ // we need to wait some time for the coordinator leader start services after election
+ waitUntil(
+ () -> coordinatorServer.getCoordinatorService().isLeader(),
+ Duration.ofSeconds(10),
+ "Coordinator leader did not recognize itself as leader");
}
// --------------------------------------------------------------------------------------------