You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2026/04/09 11:37:14 UTC

(fluss) 02/02: [server] Update leader election process and improve coordinator epoch handling

This is an automated email from the ASF dual-hosted git repository.

jark pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git

commit 2c031c480aed23bbee27401e18185de6e9d97071
Author: Jark Wu <ja...@apache.org>
AuthorDate: Thu Apr 9 15:05:09 2026 +0800

    [server] Update leader election process and improve coordinator epoch handling
---
 .../server/coordinator/CoordinatorContext.java     | 29 ++++++++++------
 .../coordinator/CoordinatorEventProcessor.java     | 16 ++++-----
 .../coordinator/CoordinatorLeaderElection.java     | 14 +++-----
 .../server/coordinator/CoordinatorServer.java      | 32 ++++--------------
 .../statemachine/ReplicaStateMachine.java          |  2 +-
 .../statemachine/TableBucketStateMachine.java      | 11 +++---
 .../fluss/server/replica/ReplicaManager.java       | 17 ++++++----
 .../java/org/apache/fluss/server/zk/ZkEpoch.java   |  7 ++++
 .../apache/fluss/server/zk/ZooKeeperClient.java    | 37 ++++++++++++++------
 .../server/coordinator/CoordinatorContextTest.java |  3 +-
 .../coordinator/CoordinatorEventProcessorTest.java |  3 +-
 .../CoordinatorHighAvailabilityITCase.java         | 33 ++++++++++++------
 .../fluss/server/coordinator/TableManagerTest.java |  7 ++--
 .../server/coordinator/TestCoordinatorContext.java | 39 ----------------------
 .../rebalance/RebalanceManagerTest.java            |  3 +-
 .../statemachine/ReplicaStateMachineTest.java      | 18 +++++-----
 .../statemachine/TableBucketStateMachineTest.java  |  9 ++---
 .../server/testutils/FlussClusterExtension.java    |  7 +++-
 18 files changed, 132 insertions(+), 155 deletions(-)

diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java
index 138ddb175..92dfc5ee5 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorContext.java
@@ -28,6 +28,7 @@ import org.apache.fluss.metadata.TablePath;
 import org.apache.fluss.server.coordinator.statemachine.BucketState;
 import org.apache.fluss.server.coordinator.statemachine.ReplicaState;
 import org.apache.fluss.server.metadata.ServerInfo;
+import org.apache.fluss.server.zk.ZkEpoch;
 import org.apache.fluss.server.zk.data.LeaderAndIsr;
 import org.apache.fluss.utils.types.Tuple2;
 
@@ -108,25 +109,33 @@ public class CoordinatorContext {
     /** A mapping from tabletServers to server tag. */
     private final Map<Integer, ServerTag> serverTags = new HashMap<>();
 
+    /**
+     * The epoch of the coordinator, which will be incremented whenever the coordinator is elected
+     * or re-elected.
+     */
+    private final int coordinatorEpoch;
+
+    /**
+     * The coordinator epoch zk version is the zk version when the coordinator epoch is updated in
+     * Zookeeper.
+     */
+    private final int coordinatorEpochZkVersion;
+
     private ServerInfo coordinatorServerInfo = null;
-    private int coordinatorEpoch = INITIAL_COORDINATOR_EPOCH;
-    private int coordinatorEpochZkVersion = INITIAL_COORDINATOR_EPOCH_ZK_VERSION;
 
-    public CoordinatorContext() {}
+    public CoordinatorContext(ZkEpoch zkEpoch) {
+        this.coordinatorEpoch = zkEpoch.getCoordinatorEpoch();
+        this.coordinatorEpochZkVersion = zkEpoch.getCoordinatorEpochZkVersion();
+    }
 
     public int getCoordinatorEpoch() {
         return coordinatorEpoch;
     }
 
-    public int getCoordinatorEpochZkVersion() {
+    public int getCoordinatorZkVersion() {
         return coordinatorEpochZkVersion;
     }
 
-    public void setCoordinatorEpochAndZkVersion(int newEpoch, int newZkVersion) {
-        this.coordinatorEpoch = newEpoch;
-        this.coordinatorEpochZkVersion = newZkVersion;
-    }
-
     public Set<String> getLiveCoordinatorServers() {
         return liveCoordinatorServers;
     }
@@ -719,8 +728,6 @@ public class CoordinatorContext {
 
     public void resetContext() {
         tablesToBeDeleted.clear();
-        coordinatorEpoch = INITIAL_COORDINATOR_EPOCH;
-        coordinatorEpochZkVersion = INITIAL_COORDINATOR_EPOCH_ZK_VERSION;
         clearTablesState();
         liveTabletServers.clear();
         liveCoordinatorServers.clear();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index 423d27622..7ae553caa 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -108,7 +108,6 @@ import org.apache.fluss.server.metadata.CoordinatorMetadataCache;
 import org.apache.fluss.server.metadata.ServerInfo;
 import org.apache.fluss.server.metrics.group.CoordinatorMetricGroup;
 import org.apache.fluss.server.utils.ServerRpcMessageUtils;
-import org.apache.fluss.server.zk.ZkEpoch;
 import org.apache.fluss.server.zk.ZooKeeperClient;
 import org.apache.fluss.server.zk.data.BucketAssignment;
 import org.apache.fluss.server.zk.data.LeaderAndIsr;
@@ -189,7 +188,6 @@ public class CoordinatorEventProcessor implements EventProcessor {
 
     public CoordinatorEventProcessor(
             ZooKeeperClient zooKeeperClient,
-            ZkEpoch zkEpoch,
             CoordinatorMetadataCache serverMetadataCache,
             CoordinatorChannelManager coordinatorChannelManager,
             CoordinatorContext coordinatorContext,
@@ -255,9 +253,6 @@ public class CoordinatorEventProcessor implements EventProcessor {
         this.ioExecutor = ioExecutor;
         this.lakeTableHelper =
                 new LakeTableHelper(zooKeeperClient, conf.getString(ConfigOptions.REMOTE_DATA_DIR));
-
-        this.coordinatorContext.setCoordinatorEpochAndZkVersion(
-                zkEpoch.getCoordinatorEpoch(), zkEpoch.getCoordinatorEpochZkVersion());
     }
 
     public CoordinatorEventManager getCoordinatorEventManager() {
@@ -314,6 +309,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
         coordinatorEventManager.close();
         rebalanceManager.close();
         onShutdown();
+        coordinatorContext.resetContext();
     }
 
     private ServerInfo getCoordinatorServerInfo() {
@@ -1626,7 +1622,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
             zooKeeperClient.updateTableAssignment(
                     tableId,
                     new TableAssignment(newTableAssignment),
-                    coordinatorContext.getCoordinatorEpochZkVersion());
+                    coordinatorContext.getCoordinatorZkVersion());
         } else {
             Map<Integer, List<Integer>> partitionAssignment =
                     coordinatorContext.getPartitionAssignment(
@@ -1639,7 +1635,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
             zooKeeperClient.updatePartitionAssignment(
                     partitionId,
                     new PartitionAssignment(tableId, newPartitionAssignment),
-                    coordinatorContext.getCoordinatorEpochZkVersion());
+                    coordinatorContext.getCoordinatorZkVersion());
         }
     }
 
@@ -1686,7 +1682,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
 
         try {
             zooKeeperClient.batchUpdateLeaderAndIsr(
-                    newLeaderAndIsrList, coordinatorContext.getCoordinatorEpochZkVersion());
+                    newLeaderAndIsrList, coordinatorContext.getCoordinatorZkVersion());
             newLeaderAndIsrList.forEach(
                     (tableBucket, newLeaderAndIsr) ->
                             result.add(new AdjustIsrResultForBucket(tableBucket, newLeaderAndIsr)));
@@ -1700,7 +1696,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
                     zooKeeperClient.updateLeaderAndIsr(
                             tableBucket,
                             newLeaderAndIsr,
-                            coordinatorContext.getCoordinatorEpochZkVersion());
+                            coordinatorContext.getCoordinatorZkVersion());
                 } catch (Exception e) {
                     LOG.error("Error when register leader and isr.", e);
                     result.add(
@@ -2227,7 +2223,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
 
         coordinatorContext.putBucketLeaderAndIsr(tableBucket, newLeaderAndIsr);
         zooKeeperClient.updateLeaderAndIsr(
-                tableBucket, newLeaderAndIsr, coordinatorContext.getCoordinatorEpochZkVersion());
+                tableBucket, newLeaderAndIsr, coordinatorContext.getCoordinatorZkVersion());
 
         coordinatorRequestBatch.newBatch();
         coordinatorRequestBatch.addNotifyLeaderRequestForTabletServers(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java
index d67f77fdf..f368e279e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorLeaderElection.java
@@ -52,8 +52,6 @@ public class CoordinatorLeaderElection implements AutoCloseable {
     private static final Logger LOG = LoggerFactory.getLogger(CoordinatorLeaderElection.class);
 
     private final String serverId;
-    private final ZooKeeperClient zkClient;
-    private final CoordinatorContext coordinatorContext;
     private final LeaderLatch leaderLatch;
     private final AtomicBoolean isLeader = new AtomicBoolean(false);
     // Cached thread pool to run leader init/cleanup callbacks outside Curator's EventThread.
@@ -66,11 +64,8 @@ public class CoordinatorLeaderElection implements AutoCloseable {
     private final AtomicReference<CompletableFuture<Void>> pendingCleanup =
             new AtomicReference<>(CompletableFuture.completedFuture(null));
 
-    public CoordinatorLeaderElection(
-            ZooKeeperClient zkClient, String serverId, CoordinatorContext coordinatorContext) {
+    public CoordinatorLeaderElection(ZooKeeperClient zkClient, String serverId) {
         this.serverId = serverId;
-        this.zkClient = zkClient;
-        this.coordinatorContext = coordinatorContext;
         this.leaderLatch =
                 new LeaderLatch(
                         zkClient.getCuratorClient(),
@@ -113,10 +108,6 @@ public class CoordinatorLeaderElection implements AutoCloseable {
                         // Run init on a separate thread to avoid deadlock with
                         // Curator's EventThread when performing ZK operations.
 
-                        // Set leader flag before init completes, so when zk found this leader, the
-                        // leader can accept requests
-                        isLeader.set(true);
-
                         leaderCallbackExecutor.execute(
                                 () -> {
                                     // Wait for any pending cleanup to finish first.
@@ -134,6 +125,9 @@ public class CoordinatorLeaderElection implements AutoCloseable {
                                     }
                                     try {
                                         initLeaderServices.run();
+                                        // Set leader flag after init completes, so when zk found
+                                        // this leader, the leader can accept requests
+                                        isLeader.set(true);
                                     } catch (CoordinatorEpochFencedException e) {
                                         LOG.warn(
                                                 "Coordinator server {} has been fenced and not become leader successfully.",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java
index a54fc2d58..f44bbd7a8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java
@@ -141,9 +141,6 @@ public class CoordinatorServer extends ServerBase {
     @Nullable
     private Authorizer authorizer;
 
-    @GuardedBy("lock")
-    private CoordinatorContext coordinatorContext;
-
     @GuardedBy("lock")
     private DynamicConfigManager dynamicConfigManager;
 
@@ -221,11 +218,9 @@ public class CoordinatorServer extends ServerBase {
                             serverId);
 
             this.zkClient = ZooKeeperUtils.startZookeeperClient(conf, this);
-            this.coordinatorContext = new CoordinatorContext();
 
             // CoordinatorLeaderElection must be created after zkClient is initialized.
-            this.coordinatorLeaderElection =
-                    new CoordinatorLeaderElection(zkClient, serverId, coordinatorContext);
+            this.coordinatorLeaderElection = new CoordinatorLeaderElection(zkClient, serverId);
 
             this.lakeCatalogDynamicLoader = new LakeCatalogDynamicLoader(conf, pluginManager, true);
             this.dynamicConfigManager = new DynamicConfigManager(zkClient, conf, true);
@@ -298,8 +293,8 @@ public class CoordinatorServer extends ServerBase {
     protected void initCoordinatorLeader() throws Exception {
         // to avoid split-brain
         ZkEpoch zkEpoch = zkClient.fenceBecomeCoordinatorLeader(serverId);
-
         registerCoordinatorLeader();
+
         synchronized (lock) {
             this.clientMetricGroup = new ClientMetricGroup(metricRegistry, SERVER_NAME);
             this.rpcClient = RpcClient.create(conf, clientMetricGroup);
@@ -311,14 +306,13 @@ public class CoordinatorServer extends ServerBase {
             autoPartitionManager.start();
 
             // start coordinator event processor after we register coordinator leader to zk
-            // so that the event processor can get the coordinator leader node from zk during start
-            // up.
-            // in HA for coordinator server, the processor also need to know the leader node during
-            // start up
+            // so that the event processor can get the coordinator leader node from zk during
+            // start up. In HA for coordinator server, the processor also need to know the leader
+            // node during start up
+            CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
             this.coordinatorEventProcessor =
                     new CoordinatorEventProcessor(
                             zkClient,
-                            zkEpoch,
                             metadataCache,
                             coordinatorChannelManager,
                             coordinatorContext,
@@ -402,11 +396,6 @@ public class CoordinatorServer extends ServerBase {
                 LOG.warn("Failed to close client metric group", t);
             }
 
-            // Reset coordinator context for next election
-            if (coordinatorContext != null) {
-                coordinatorContext.resetContext();
-            }
-
             LOG.info("Coordinator leader services cleaned up successfully.");
         }
     }
@@ -601,15 +590,6 @@ public class CoordinatorServer extends ServerBase {
                 exception = ExceptionUtils.firstOrSuppressed(t, exception);
             }
 
-            try {
-                if (coordinatorContext != null) {
-                    // then reset coordinatorContext
-                    coordinatorContext.resetContext();
-                }
-            } catch (Throwable t) {
-                exception = ExceptionUtils.firstOrSuppressed(t, exception);
-            }
-
             try {
                 if (lakeTableTieringManager != null) {
                     lakeTableTieringManager.close();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java
index 2416b163b..63f5dd965 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachine.java
@@ -488,7 +488,7 @@ public class ReplicaStateMachine {
         }
         try {
             zooKeeperClient.batchUpdateLeaderAndIsr(
-                    toUpdateLeaderAndIsrList, coordinatorContext.getCoordinatorEpochZkVersion());
+                    toUpdateLeaderAndIsrList, coordinatorContext.getCoordinatorZkVersion());
             toUpdateLeaderAndIsrList.forEach(coordinatorContext::putBucketLeaderAndIsr);
             return adjustedLeaderAndIsr;
         } catch (Exception e) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java
index b01e65c92..300e98f08 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachine.java
@@ -312,9 +312,7 @@ public class TableBucketStateMachine {
             LeaderAndIsr leaderAndIsr = electionResult.leaderAndIsr;
             try {
                 zooKeeperClient.registerLeaderAndIsr(
-                        tableBucket,
-                        leaderAndIsr,
-                        coordinatorContext.getCoordinatorEpochZkVersion());
+                        tableBucket, leaderAndIsr, coordinatorContext.getCoordinatorZkVersion());
             } catch (Exception e) {
                 LOG.error(
                         "Fail to create state node for table bucket {} in zookeeper.",
@@ -382,8 +380,7 @@ public class TableBucketStateMachine {
         if (!tableBucketLeadAndIsrInfos.isEmpty()) {
             try {
                 zooKeeperClient.batchRegisterLeaderAndIsrForTablePartition(
-                        tableBucketLeadAndIsrInfos,
-                        coordinatorContext.getCoordinatorEpochZkVersion());
+                        tableBucketLeadAndIsrInfos, coordinatorContext.getCoordinatorZkVersion());
                 registerSuccessList.addAll(tableBucketLeadAndIsrInfos);
             } catch (Exception e) {
                 LOG.error(
@@ -461,7 +458,7 @@ public class TableBucketStateMachine {
                 zooKeeperClient.registerLeaderAndIsr(
                         info.getTableBucket(),
                         info.getLeaderAndIsr(),
-                        coordinatorContext.getCoordinatorEpochZkVersion());
+                        coordinatorContext.getCoordinatorZkVersion());
                 registerSuccessList.add(info);
             } catch (Exception e) {
                 LOG.error(
@@ -506,7 +503,7 @@ public class TableBucketStateMachine {
             zooKeeperClient.updateLeaderAndIsr(
                     tableBucket,
                     electionResult.leaderAndIsr,
-                    coordinatorContext.getCoordinatorEpochZkVersion());
+                    coordinatorContext.getCoordinatorZkVersion());
         } catch (Exception e) {
             LOG.error(
                     "Fail to update bucket LeaderAndIsr for table bucket {}.",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index 6960b5191..19ab3f936 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -336,6 +336,11 @@ public class ReplicaManager implements ServerReconfigurable {
         return minInSyncReplicas;
     }
 
+    @VisibleForTesting
+    public int getCoordinatorEpoch() {
+        return coordinatorEpoch;
+    }
+
     // ============ ServerReconfigurable Implementation ============
 
     @Override
@@ -1879,9 +1884,14 @@ public class ReplicaManager implements ServerReconfigurable {
                             requestCoordinatorEpoch, requestName, this.coordinatorEpoch);
             LOG.warn("Ignore the {} request because {}", requestName, errorMessage);
             throw new InvalidCoordinatorException(errorMessage);
-        } else {
+        } else if (requestCoordinatorEpoch > this.coordinatorEpoch) {
+            LOG.info(
+                    "Update coordinator epoch from {} to {} for coordinator leader switch.",
+                    this.coordinatorEpoch,
+                    requestCoordinatorEpoch);
             this.coordinatorEpoch = requestCoordinatorEpoch;
         }
+        // ignore equal case
     }
 
     private void dropEmptyTableOrPartitionDir(Path dir, long id, String dirType) {
@@ -1991,11 +2001,6 @@ public class ReplicaManager implements ServerReconfigurable {
         return serverMetricGroup;
     }
 
-    @VisibleForTesting
-    public void resetCoordinatorEpoch() {
-        this.coordinatorEpoch = CoordinatorContext.INITIAL_COORDINATOR_EPOCH;
-    }
-
     /**
      * Interface to represent the state of hosted {@link Replica}. We create a concrete (active)
      * {@link Replica} instance when the TabletServer receives a createLogLeader request or
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java
index f3d401a25..3f3984373 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZkEpoch.java
@@ -18,8 +18,15 @@
 
 package org.apache.fluss.server.zk;
 
+import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH;
+import static org.apache.fluss.server.coordinator.CoordinatorContext.INITIAL_COORDINATOR_EPOCH_ZK_VERSION;
+
 /** Class for coordinator epoch and coordinator epoch zk version. */
 public class ZkEpoch {
+
+    public static final ZkEpoch INITIAL_EPOCH =
+            new ZkEpoch(INITIAL_COORDINATOR_EPOCH, INITIAL_COORDINATOR_EPOCH_ZK_VERSION);
+
     private final int coordinatorEpoch;
     private final int coordinatorEpochZkVersion;
 
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
index 3c8b58fa2..cb34a2f2e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
@@ -139,24 +139,39 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull;
  *
  * <p>In some method, 'expectedZkVersion' is used to execute an epoch Zookeeper version check.
  * Conditions requiring epoch checks (all must be met):
- * ┌─────────────────────────────────────────────────────────┐ │ 1. Invoked by the Coordinator (not
- * the TabletServer) │ │ 2. Operates on persistent nodes (not ephemeral) │ │ 3. Constitutes a
- * "control plane" operation: │ │ partition assignment or LeaderAndIsr election │ │ 4. Concurrent
- * access to the same path by old and new │ │ leaders during leader failover │ │ 5. No other
- * mechanisms (optimistic locking, │ │ idempotency, or reloading) provide fallback │
+ *
+ * <pre>
+ * ┌─────────────────────────────────────────────────────────┐
+ * │ 1. Invoked by the Coordinator (not the TabletServer)    │
+ * │ 2. Operates on persistent nodes (not ephemeral)         │
+ * │ 3. Constitutes a "control plane" operation:             │
+ * │    partition assignment or LeaderAndIsr election        │
+ * │ 4. Concurrent access to the same path by old and new    │
+ * │    leaders during leader failover                       │
+ * │ 5. No other mechanisms (optimistic locking,             │
+ * │    idempotency, or reloading) provide fallback          │
  * └─────────────────────────────────────────────────────────┘
+ * </pre>
+ *
+ * <p>In practice, only two types of operations truly require this:
  *
- * <p>In practice, only two types of operations truly require this: - CRUD for Table/Partition
- * Assignment (assignment decisions). - CRUD for LeaderAndIsr (leader election results).
+ * <ul>
+ *   <li>CRUD for Table/Partition Assignment (assignment decisions).
+ *   <li>CRUD for LeaderAndIsr (leader election results).
+ * </ul>
  *
  * <p>These operations are inevitably executed concurrently by the old and new coordinators during
  * failover (as the new leader immediately reassigns partitions), and overwrites cannot be
  * automatically recovered.
  *
- * <p>All other operations do not require epoch checks because: - DDL operations are protected
- * against concurrency by client reconnection mechanisms. - TabletServer operations are unaffected
- * by coordinator failovers. - ACLs and Configs have their own version control or idempotency
- * guarantees. - Ephemeral nodes are managed via session lifecycle.
+ * <p>All other operations do not require epoch checks because:
+ *
+ * <ul>
+ *   <li>DDL operations are protected against concurrency by client reconnection mechanisms.
+ *   <li>TabletServer operations are unaffected by coordinator failovers.
+ *   <li>ACLs and Configs have their own version control or idempotency guarantees.
+ *   <li>Ephemeral nodes are managed via session lifecycle.
+ * </ul>
  */
 @Internal
 public class ZooKeeperClient implements AutoCloseable {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java
index 5c321dfff..2b788bdb4 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorContextTest.java
@@ -22,6 +22,7 @@ import org.apache.fluss.metadata.Schema;
 import org.apache.fluss.metadata.TableDescriptor;
 import org.apache.fluss.metadata.TableInfo;
 import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.server.zk.ZkEpoch;
 import org.apache.fluss.types.DataTypes;
 
 import org.junit.jupiter.api.Test;
@@ -37,7 +38,7 @@ class CoordinatorContextTest {
 
     @Test
     void testGetLakeTableCount() {
-        CoordinatorContext context = new CoordinatorContext();
+        CoordinatorContext context = new CoordinatorContext(ZkEpoch.INITIAL_EPOCH);
 
         // Initially, there should be no tables
         assertThat(context.allTables()).isEmpty();
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
index cd036b364..63f0b5fd9 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
@@ -1061,10 +1061,9 @@ class CoordinatorEventProcessorTest {
     private CoordinatorEventProcessor buildCoordinatorEventProcessor() {
         return new CoordinatorEventProcessor(
                 zookeeperClient,
-                zkEpoch,
                 serverMetadataCache,
                 testCoordinatorChannelManager,
-                new TestCoordinatorContext(),
+                new CoordinatorContext(zkEpoch),
                 autoPartitionManager,
                 lakeTableTieringManager,
                 TestingMetricGroups.COORDINATOR_METRICS,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java
index e974ec2d9..68e46e359 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorHighAvailabilityITCase.java
@@ -283,16 +283,14 @@ class CoordinatorHighAvailabilityITCase {
                 .isEqualTo(standby.getServerId());
 
         int newCoordinatorEpoch = standby.getCoordinatorEventProcessor().getCoordinatorEpoch();
+        // wait the new coordinator notifies the tablet server about the new coordinator epoch
+        waitUntil(
+                () -> tabletServer.getReplicaManager().getCoordinatorEpoch() == newCoordinatorEpoch,
+                Duration.ofSeconds(30),
+                "Tablet server did not update to new coordinator epoch");
 
         TabletServerGateway tsGateway = createGatewayForTabletServer(tabletServer);
 
-        // Send request with new coordinator epoch first to ensure the tablet server
-        // has updated its stored epoch to the latest value
-        tsGateway
-                .updateMetadata(
-                        new UpdateMetadataRequest().setCoordinatorEpoch(newCoordinatorEpoch))
-                .get();
-
         // Send request with old coordinator epoch — tablet server should reject it
         assertThatThrownBy(
                         () ->
@@ -339,11 +337,11 @@ class CoordinatorHighAvailabilityITCase {
         int newLeaderEpochZkVersion =
                 standby.getCoordinatorEventProcessor()
                         .getCoordinatorContext()
-                        .getCoordinatorEpochZkVersion();
+                        .getCoordinatorZkVersion();
         int oldLeaderEpochZkVersion =
                 leader.getCoordinatorEventProcessor()
                         .getCoordinatorContext()
-                        .getCoordinatorEpochZkVersion();
+                        .getCoordinatorZkVersion();
 
         assertThatThrownBy(
                         () ->
@@ -413,11 +411,23 @@ class CoordinatorHighAvailabilityITCase {
         return configuration;
     }
 
-    private void waitUntilCoordinatorServerElected() {
+    private void waitUntilCoordinatorServerElected() throws Exception {
         waitUntil(
                 () -> zookeeperClient.getCoordinatorLeaderAddress().isPresent(),
                 Duration.ofMinutes(1),
                 "Fail to wait coordinator server elected");
+        waitUntilCoordinatorLeaderReady();
+    }
+
+    private void waitUntilCoordinatorLeaderReady() throws Exception {
+        CoordinatorAddress currentLeaderAddress =
+                zookeeperClient.getCoordinatorLeaderAddress().get();
+        CoordinatorServer currentLeader = findServerById(currentLeaderAddress.getId());
+        // we need to wait some time for the coordinator leader start services after election
+        waitUntil(
+                () -> currentLeader.getCoordinatorService().isLeader(),
+                Duration.ofSeconds(30),
+                "Coordinator leader did not recognize itself as leader");
     }
 
     private CoordinatorServer findServerById(String serverId) {
@@ -493,7 +503,7 @@ class CoordinatorHighAvailabilityITCase {
     }
 
     /** Waits until a new coordinator leader is elected that is different from the given old one. */
-    private void waitUntilNewLeaderElected(String oldLeaderId) {
+    private void waitUntilNewLeaderElected(String oldLeaderId) throws Exception {
         waitUntil(
                 () -> {
                     try {
@@ -507,6 +517,7 @@ class CoordinatorHighAvailabilityITCase {
                 },
                 Duration.ofMinutes(1),
                 "Fail to wait for new coordinator leader to be elected");
+        waitUntilCoordinatorLeaderReady();
     }
 
     /** Verifies that the given server can process requests as a leader. */
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
index 2c30fc748..f5357c549 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
@@ -31,6 +31,7 @@ import org.apache.fluss.server.coordinator.statemachine.TableBucketStateMachine;
 import org.apache.fluss.server.entity.DeleteReplicaResultForBucket;
 import org.apache.fluss.server.metadata.ServerInfo;
 import org.apache.fluss.server.zk.NOPErrorHandler;
+import org.apache.fluss.server.zk.ZkEpoch;
 import org.apache.fluss.server.zk.ZooKeeperClient;
 import org.apache.fluss.server.zk.ZooKeeperExtension;
 import org.apache.fluss.server.zk.data.BucketAssignment;
@@ -78,6 +79,7 @@ class TableManagerTest {
             new AllCallbackWrapper<>(new ZooKeeperExtension());
 
     private static ZooKeeperClient zookeeperClient;
+    private static ZkEpoch zkEpoch;
     private static ExecutorService ioExecutor;
 
     private CoordinatorContext coordinatorContext;
@@ -86,11 +88,12 @@ class TableManagerTest {
     private TestCoordinatorChannelManager testCoordinatorChannelManager;
 
     @BeforeAll
-    static void baseBeforeAll() {
+    static void baseBeforeAll() throws Exception {
         zookeeperClient =
                 ZOO_KEEPER_EXTENSION_WRAPPER
                         .getCustomExtension()
                         .getZooKeeperClient(NOPErrorHandler.INSTANCE);
+        zkEpoch = zookeeperClient.fenceBecomeCoordinatorLeader("1");
         ioExecutor = Executors.newFixedThreadPool(1);
     }
 
@@ -113,7 +116,7 @@ class TableManagerTest {
 
     private void initTableManager() {
         testingEventManager = new TestingEventManager();
-        coordinatorContext = new TestCoordinatorContext();
+        coordinatorContext = new CoordinatorContext(zkEpoch);
         testCoordinatorChannelManager = new TestCoordinatorChannelManager();
         Configuration conf = new Configuration();
         conf.setString(ConfigOptions.REMOTE_DATA_DIR, "/tmp/fluss/remote-data");
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorContext.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorContext.java
deleted file mode 100644
index b931d564f..000000000
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TestCoordinatorContext.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.fluss.server.coordinator;
-
-import org.apache.fluss.server.zk.data.ZkVersion;
-
-/** A coordinator context for test purpose which can set epoch manually. */
-public class TestCoordinatorContext extends CoordinatorContext {
-    public TestCoordinatorContext() {
-        // When create or modify ZooKeeper node, it should check ZooKeeper epoch node version
-        // to ensure the coordinator is still holding the leadership. However, in the test
-        // cases, we don't register epoch node, so we skip the check process by setting
-        // "coordinatorEpochZkVersion" to ZkVersion.MATCH_ANY_VERSION
-        super();
-        this.setCoordinatorEpochAndZkVersion(
-                INITIAL_COORDINATOR_EPOCH, ZkVersion.MATCH_ANY_VERSION.getVersion());
-    }
-
-    public TestCoordinatorContext(int coordinatorEpoch, int coordinatorEpochZkVersion) {
-        super();
-        this.setCoordinatorEpochAndZkVersion(coordinatorEpoch, coordinatorEpochZkVersion);
-    }
-}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java
index 57ba9fff7..6ae36b8de 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/rebalance/RebalanceManagerTest.java
@@ -137,10 +137,9 @@ public class RebalanceManagerTest {
     private CoordinatorEventProcessor buildCoordinatorEventProcessor() {
         return new CoordinatorEventProcessor(
                 zookeeperClient,
-                zkEpoch,
                 serverMetadataCache,
                 testCoordinatorChannelManager,
-                new CoordinatorContext(),
+                new CoordinatorContext(zkEpoch),
                 autoPartitionManager,
                 lakeTableTieringManager,
                 TestingMetricGroups.COORDINATOR_METRICS,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java
index 4c89b5722..f1ad2fe1c 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/ReplicaStateMachineTest.java
@@ -30,11 +30,11 @@ import org.apache.fluss.server.coordinator.CoordinatorContext;
 import org.apache.fluss.server.coordinator.CoordinatorRequestBatch;
 import org.apache.fluss.server.coordinator.CoordinatorTestUtils;
 import org.apache.fluss.server.coordinator.TestCoordinatorChannelManager;
-import org.apache.fluss.server.coordinator.TestCoordinatorContext;
 import org.apache.fluss.server.coordinator.event.DeleteReplicaResponseReceivedEvent;
 import org.apache.fluss.server.entity.DeleteReplicaResultForBucket;
 import org.apache.fluss.server.metadata.ServerInfo;
 import org.apache.fluss.server.zk.NOPErrorHandler;
+import org.apache.fluss.server.zk.ZkEpoch;
 import org.apache.fluss.server.zk.ZooKeeperClient;
 import org.apache.fluss.server.zk.ZooKeeperExtension;
 import org.apache.fluss.server.zk.data.LeaderAndIsr;
@@ -71,18 +71,20 @@ class ReplicaStateMachineTest {
             new AllCallbackWrapper<>(new ZooKeeperExtension());
 
     private static ZooKeeperClient zookeeperClient;
+    private static ZkEpoch zkEpoch;
 
     @BeforeAll
-    static void baseBeforeAll() {
+    static void baseBeforeAll() throws Exception {
         zookeeperClient =
                 ZOO_KEEPER_EXTENSION_WRAPPER
                         .getCustomExtension()
                         .getZooKeeperClient(NOPErrorHandler.INSTANCE);
+        zkEpoch = zookeeperClient.fenceBecomeCoordinatorLeader("1");
     }
 
     @Test
     void testStartup() {
-        CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+        CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
 
         // init coordinator server context with a table assignment
         TableBucket tableBucket = new TableBucket(1, 0);
@@ -107,7 +109,7 @@ class ReplicaStateMachineTest {
 
     @Test
     void testReplicaStateChange() {
-        CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+        CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
         ReplicaStateMachine replicaStateMachine = createReplicaStateMachine(coordinatorContext);
 
         // test check valid replica state change
@@ -136,7 +138,7 @@ class ReplicaStateMachineTest {
     @Test
     void testDeleteReplicaStateChange() {
         Map<TableBucketReplica, Boolean> isReplicaDeleteSuccess = new HashMap<>();
-        CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+        CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
         coordinatorContext.setLiveTabletServers(
                 CoordinatorTestUtils.createServers(Arrays.asList(0, 1)));
         // use a context that will return a gateway that always get success ack
@@ -167,7 +169,7 @@ class ReplicaStateMachineTest {
         }
 
         // now, we change a context that some gateway will return exception
-        coordinatorContext = new CoordinatorContext();
+        coordinatorContext = new CoordinatorContext(zkEpoch);
         coordinatorContext.setLiveTabletServers(
                 CoordinatorTestUtils.createServers(Arrays.asList(0, 1)));
         coordinatorContext.putBucketLeaderAndIsr(tableBucket1, new LeaderAndIsr(0, 0));
@@ -193,7 +195,7 @@ class ReplicaStateMachineTest {
 
     @Test
     void testOfflineReplicasShouldBeRemovedFromIsr() throws Exception {
-        CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+        CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
         coordinatorContext.setLiveTabletServers(createServers(new int[] {0, 1, 2}));
         ReplicaStateMachine replicaStateMachine = createReplicaStateMachine(coordinatorContext);
 
@@ -233,7 +235,7 @@ class ReplicaStateMachineTest {
 
     @Test
     void testOfflineReplicaShouldBeRemovedFromIsr() throws Exception {
-        CoordinatorContext coordinatorContext = new TestCoordinatorContext();
+        CoordinatorContext coordinatorContext = new CoordinatorContext(zkEpoch);
         coordinatorContext.setLiveTabletServers(createServers(new int[] {0, 1, 2}));
         ReplicaStateMachine replicaStateMachine = createReplicaStateMachine(coordinatorContext);
 
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java
index d6697f8cb..2ea3718ff 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/statemachine/TableBucketStateMachineTest.java
@@ -34,7 +34,6 @@ import org.apache.fluss.server.coordinator.LakeCatalogDynamicLoader;
 import org.apache.fluss.server.coordinator.LakeTableTieringManager;
 import org.apache.fluss.server.coordinator.MetadataManager;
 import org.apache.fluss.server.coordinator.TestCoordinatorChannelManager;
-import org.apache.fluss.server.coordinator.TestCoordinatorContext;
 import org.apache.fluss.server.coordinator.event.CoordinatorEventManager;
 import org.apache.fluss.server.coordinator.lease.KvSnapshotLeaseManager;
 import org.apache.fluss.server.coordinator.statemachine.ReplicaLeaderElection.ControlledShutdownLeaderElection;
@@ -57,7 +56,6 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
-import java.io.IOException;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
@@ -106,12 +104,12 @@ class TableBucketStateMachineTest {
     }
 
     @BeforeEach
-    void beforeEach() throws IOException {
+    void beforeEach() {
         Configuration conf = new Configuration();
         conf.setString(ConfigOptions.COORDINATOR_HOST, "localhost");
         String remoteDir = "/tmp/fluss/remote-data";
         conf.setString(ConfigOptions.REMOTE_DATA_DIR, remoteDir);
-        coordinatorContext = new TestCoordinatorContext();
+        coordinatorContext = new CoordinatorContext(zkEpoch);
         testCoordinatorChannelManager = new TestCoordinatorChannelManager();
         coordinatorRequestBatch =
                 new CoordinatorRequestBatch(
@@ -233,8 +231,6 @@ class TableBucketStateMachineTest {
         coordinatorContext.putTablePath(tableId, fakeTablePath);
         coordinatorContext.updateBucketReplicaAssignment(tableBucket, Arrays.asList(0, 1, 2));
         coordinatorContext.putBucketState(tableBucket, NewBucket);
-        coordinatorContext.setCoordinatorEpochAndZkVersion(
-                0, ZkVersion.MATCH_ANY_VERSION.getVersion());
         // case1: init a new leader for NewBucket to OnlineBucket
         tableBucketStateMachine.handleStateChange(Collections.singleton(tableBucket), OnlineBucket);
         // non any alive servers, the state change fail
@@ -280,7 +276,6 @@ class TableBucketStateMachineTest {
         CoordinatorEventProcessor coordinatorEventProcessor =
                 new CoordinatorEventProcessor(
                         zookeeperClient,
-                        zkEpoch,
                         serverMetadataCache,
                         new CoordinatorChannelManager(
                                 RpcClient.create(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
index 9498419e0..74a041596 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
@@ -942,11 +942,16 @@ public final class FlussClusterExtension
         return coordinatorServer;
     }
 
-    public void waitUntilCoordinatorServerElected() throws Exception {
+    private void waitUntilCoordinatorServerElected() {
         waitUntil(
                 () -> zooKeeperClient.getCoordinatorLeaderAddress().isPresent(),
                 Duration.ofSeconds(10),
                 "Fail to wait coordinator server elected");
+        // we need to wait some time for the coordinator leader start services after election
+        waitUntil(
+                () -> coordinatorServer.getCoordinatorService().isLeader(),
+                Duration.ofSeconds(10),
+                "Coordinator leader did not recognize itself as leader");
     }
 
     // --------------------------------------------------------------------------------------------