You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by am...@apache.org on 2026/10/08 08:27:56 UTC

(ratis) branch master updated: RATIS-2726. Fix testFollowerHeartbeatMetric failure after leader step down. (#1621)

This is an automated email from the ASF dual-hosted git repository.

amaliujia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git


The following commit(s) were added to refs/heads/master by this push:
     new 1c2f23488 RATIS-2726. Fix testFollowerHeartbeatMetric failure after leader step down. (#1621)
1c2f23488 is described below

commit 1c2f23488b1ef357ef4523823d201acd67f03406
Author: slfan1989 <55...@users.noreply.github.com>
AuthorDate: Thu Oct 8 16:27:51 2026 +0800

    RATIS-2726. Fix testFollowerHeartbeatMetric failure after leader step down. (#1621)
---
 .../apache/ratis/server/impl/LeaderStateImpl.java  |  19 +++-
 .../server/metrics/RaftServerMetricsImpl.java      |  12 +++
 .../java/org/apache/ratis/LogAppenderTests.java    | 112 ++++++++++++++-------
 3 files changed, 104 insertions(+), 39 deletions(-)

diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
index 798fb3c4f..b3f872e2b 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
@@ -351,6 +351,8 @@ class LeaderStateImpl implements LeaderState {
   private final int stagingCatchupGap;
   private final TimeDuration stagingTimeout;
   private final RaftServerMetricsImpl raftServerMetrics;
+  // Serialize follower metric registration and updates with cleanup on step-down.
+  private final Object followerMetricsLock = new Object();
   private final LogAppenderMetrics logAppenderMetrics;
   private final long followerMaxGapThreshold;
   private final PendingStepDown pendingStepDown;
@@ -483,7 +485,9 @@ class LeaderStateImpl implements LeaderState {
       replyFlusher.stop();
     }
     logAppenderMetrics.unregister();
-    raftServerMetrics.unregister();
+    synchronized (followerMetricsLock) {
+      raftServerMetrics.clearFollowerHeartbeatMetrics();
+    }
     pendingRequests.close();
     watchRequests.close();
     leaderTracer.close();
@@ -687,7 +691,11 @@ class LeaderStateImpl implements LeaderState {
     final List<LogAppender> newAppenders = newPeers.stream().map(peer -> {
       final FollowerInfo f = new FollowerInfoImpl(server.getMemberId(), peer, this::getPeer, t, nextIndex, caughtUp);
       followerInfoMap.put(peer.getId(), f);
-      raftServerMetrics.addFollower(peer.getId());
+      synchronized (followerMetricsLock) {
+        if (!isStopped.get()) {
+          raftServerMetrics.addFollower(peer.getId());
+        }
+      }
       logAppenderMetrics.addFollowerGauges(peer.getId(), f::getNextIndex, f::getMatchIndex, f::getLastRpcTime);
       return newLogAppender(f);
     }).collect(Collectors.toList());
@@ -1370,7 +1378,12 @@ class LeaderStateImpl implements LeaderState {
       server.getStateMachine().leaderEvent().notifyFollowerSlowness(leaderInfo, follower.getPeer());
     }
     final RaftPeerId followerId = follower.getId();
-    raftServerMetrics.recordFollowerHeartbeatElapsedTime(followerId, elapsedTime.toLong(TimeUnit.NANOSECONDS));
+    synchronized (followerMetricsLock) {
+      // An appender may finish its current iteration after stopAsync() was called.
+      if (!isStopped.get()) {
+        raftServerMetrics.recordFollowerHeartbeatElapsedTime(followerId, elapsedTime.toLong(TimeUnit.NANOSECONDS));
+      }
+    }
   }
 
   @Override
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java
index 70711c3e3..0b6af9726 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java
@@ -196,6 +196,18 @@ public final class RaftServerMetricsImpl extends RatisMetrics implements RaftSer
     addPeerCommitIndexGauge(followerName);
   }
 
+  /**
+   * Remove the heartbeat gauges owned by the leader, retaining server metrics.
+   * Peer commit-index gauges still read the server cache, which is updated as a follower.
+   * The caller must serialize this with follower metric registration and updates.
+   */
+  public void clearFollowerHeartbeatMetrics() {
+    followerLastHeartbeatElapsedTimeMap.keySet().forEach(followerId -> {
+      getRegistry().remove(String.format(FOLLOWER_LAST_HEARTBEAT_ELAPSED_TIME_METRIC, followerId));
+    });
+    followerLastHeartbeatElapsedTimeMap.clear();
+  }
+
   /**
    * Register a commit index tracker for the peer in cluster.
    */
diff --git a/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java b/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java
index 624ce4757..546f1dafb 100644
--- a/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java
+++ b/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java
@@ -18,11 +18,15 @@
 package org.apache.ratis;
 
 import static org.apache.ratis.RaftTestUtil.waitForLeader;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import org.apache.ratis.RaftTestUtil.SimpleMessage;
 import org.apache.ratis.client.RaftClient;
 import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.metrics.MetricRegistries;
 import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl;
 import org.apache.ratis.metrics.impl.DefaultTimekeeperImpl;
 import org.apache.ratis.proto.RaftProtos.LogEntryProto;
@@ -48,7 +52,9 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.EnumMap;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.SortedMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -128,50 +134,84 @@ public abstract class LogAppenderTests<CLUSTER extends MiniRaftCluster>
   }
 
   @Test
-  public void testFollowerHeartbeatMetric() throws IOException, InterruptedException {
-
-    // Start a 3 node Ratis ring.
+  public void testFollowerHeartbeatMetric() throws Exception {
     final MiniRaftCluster cluster = newCluster(3);
-    cluster.start();
-    final RaftServer.Division leaderServer = waitForLeader(cluster);
+    final Map<RaftPeerId, RatisMetricRegistryImpl> registries = new HashMap<>();
+    try {
+      cluster.start();
+      final RaftPeerId originalLeader = waitForLeader(cluster).getId();
+      for (RaftServer.Division server : cluster.iterateDivisions()) {
+        registries.put(server.getId(), (RatisMetricRegistryImpl)
+            ((RaftServerMetricsImpl) server.getRaftServerMetrics()).getRegistry());
+      }
 
-    // Write 10 messages to leader.
-    try(RaftClient client = cluster.createClient(leaderServer.getId())) {
-      for (int i = 1; i <= 10; i++) {
-        client.io().send(new RaftTestUtil.SimpleMessage("Msg to make leader ready " +  i));
+      // Check initial leadership, step-down, and re-election without restarting the servers.
+      for (int round = 0; round < 3; round++) {
+        final RaftPeerId leaderId = waitForLeader(cluster).getId();
+        try (RaftClient client = cluster.createClient(leaderId)) {
+          for (int i = 0; i < 10; i++) {
+            RaftTestUtil.assertSuccessReply(
+                client.io().send(new SimpleMessage("heartbeat metrics round " + round + " message " + i)));
+          }
+          JavaUtils.attempt(() -> assertFollowerHeartbeatMetrics(cluster, registries),
+              100, HUNDRED_MILLIS, "check heartbeat metrics", LOG);
+          final RatisMetricRegistryImpl previousLeaderRegistry = registries.get(leaderId);
+          final SortedMap<String, Gauge> commitIndexGauges = previousLeaderRegistry.getGauges(
+              (s, m) -> s.endsWith("_peerCommitIndex"));
+          assertEquals(3, commitIndexGauges.size());
+          if (round < 2) {
+            final RaftPeerId nextLeader = round == 0 ? cluster.getFollowers().get(0).getId() : originalLeader;
+            assertTrue(client.admin().transferLeadership(nextLeader, 20_000).isSuccess());
+            assertEquals(nextLeader, waitForLeader(cluster).getId());
+            // Commit-index gauges read the server cache and remain useful after step-down.
+            final SortedMap<String, Gauge> retainedGauges = previousLeaderRegistry.getGauges(
+                (s, m) -> s.endsWith("_peerCommitIndex"));
+            commitIndexGauges.forEach((name, gauge) -> assertSame(gauge, retainedGauges.get(name)));
+          }
+        }
       }
-    } catch (IOException e) {
-      throw e;
+    } finally {
+      cluster.shutdown();
     }
+    registries.values().forEach(registry -> assertTrue(
+        !MetricRegistries.global().get(registry.getMetricRegistryInfo()).isPresent(),
+        "Server registry should be unregistered after shutdown"));
+  }
 
-    final RatisMetricRegistryImpl ratisMetricRegistry = (RatisMetricRegistryImpl)
-        ((RaftServerMetricsImpl)leaderServer.getRaftServerMetrics()).getRegistry();
-
-    // Get all last_heartbeat_elapsed_time metric gauges. Should be equal to number of followers.
-    SortedMap<String, Gauge> heartbeatElapsedTimeGauges = ratisMetricRegistry.getGauges((s, metric) ->
+  private void assertFollowerHeartbeatMetrics(MiniRaftCluster cluster,
+      Map<RaftPeerId, RatisMetricRegistryImpl> registries) {
+    final RaftServer.Division leader = cluster.getLeader();
+    assertNotNull(leader);
+    final RatisMetricRegistryImpl leaderRegistry = registries.get(leader.getId());
+    final SortedMap<String, Gauge> heartbeatGauges = leaderRegistry.getGauges((s, m) ->
         s.contains("lastHeartbeatElapsedTime"));
-    assertTrue(heartbeatElapsedTimeGauges.size() == 2);
-
-    for (RaftServer.Division followerServer : cluster.getFollowers()) {
-      String followerId = followerServer.getId().toString();
-      Gauge metric = heartbeatElapsedTimeGauges.entrySet().parallelStream().filter(e -> e.getKey().contains(
-          followerId)).iterator().next().getValue();
-      // Metric for this follower exists.
-      assertTrue(metric != null);
-      // Metric in nanos > 0.
-      assertTrue((long)metric.getValue() > 0);
-      // Try to get Heartbeat metrics for follower.
-      final RaftServerMetricsImpl followerMetrics = (RaftServerMetricsImpl) followerServer.getRaftServerMetrics();
-      // Metric should not exist. It only exists in leader.
-      final RatisMetricRegistryImpl followerMetricRegistry = (RatisMetricRegistryImpl)followerMetrics.getRegistry();
-      assertTrue(followerMetricRegistry.getGauges((s, m) -> s.contains("lastHeartbeatElapsedTime")).isEmpty());
-      for (boolean heartbeat : new boolean[] { true, false }) {
-        final DefaultTimekeeperImpl t = (DefaultTimekeeperImpl) followerMetrics.getFollowerAppendEntryTimer(heartbeat);
-        assertTrue(t.getTimer().getMeanRate() > 0.0d);
-        assertTrue(t.getTimer().getCount() > 0L);
+    assertEquals(2, heartbeatGauges.size());
+
+    for (RaftServer.Division server : cluster.iterateDivisions()) {
+      final RatisMetricRegistryImpl registry = registries.get(server.getId());
+      assertSame(registry, MetricRegistries.global().get(registry.getMetricRegistryInfo())
+          .orElseThrow(() -> new AssertionError("Missing server registry for " + server.getId())));
+      assertTrue(!registry.getGauges((s, m) -> s.endsWith(server.getId() + "_peerCommitIndex")).isEmpty());
+      assertEquals(5, registry.getGauges((s, m) -> s.contains("retryCache")).size());
+
+      if (server.getId().equals(leader.getId())) {
+        continue;
+      }
+      final Gauge<?> heartbeat = heartbeatGauges.entrySet().stream()
+          .filter(e -> e.getKey().endsWith(server.getId() + "_lastHeartbeatElapsedTime"))
+          .findFirst().orElseThrow(() -> new AssertionError("Missing heartbeat gauge for " + server.getId()))
+          .getValue();
+      assertTrue((Long) heartbeat.getValue() > 0);
+      assertTrue(registry.getGauges((s, m) -> s.contains("lastHeartbeatElapsedTime")).isEmpty());
+      assertTrue(registry.getGauges((s, m) -> s.endsWith("numPendingRequestInQueue")
+          || s.endsWith("numPendingRequestMegaByteSize") || s.matches(".*numWatch.*RequestInQueue")).isEmpty());
+      final RaftServerMetricsImpl metrics = (RaftServerMetricsImpl) server.getRaftServerMetrics();
+      for (boolean isHeartbeat : new boolean[] {true, false}) {
+        final DefaultTimekeeperImpl timer = (DefaultTimekeeperImpl) metrics.getFollowerAppendEntryTimer(isHeartbeat);
+        assertTrue(timer.getTimer().getMeanRate() > 0.0d);
+        assertTrue(timer.getTimer().getCount() > 0L);
       }
     }
-    cluster.shutdown();
   }
 
   void runTest(CLUSTER cluster) throws Exception {