You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ratis.apache.org by am...@apache.org on 2026/10/08 08:27:56 UTC
(ratis) branch master updated: RATIS-2726. Fix testFollowerHeartbeatMetric failure after leader step down. (#1621)
This is an automated email from the ASF dual-hosted git repository.
amaliujia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ratis.git
The following commit(s) were added to refs/heads/master by this push:
new 1c2f23488 RATIS-2726. Fix testFollowerHeartbeatMetric failure after leader step down. (#1621)
1c2f23488 is described below
commit 1c2f23488b1ef357ef4523823d201acd67f03406
Author: slfan1989 <55...@users.noreply.github.com>
AuthorDate: Thu Oct 8 16:27:51 2026 +0800
RATIS-2726. Fix testFollowerHeartbeatMetric failure after leader step down. (#1621)
---
.../apache/ratis/server/impl/LeaderStateImpl.java | 19 +++-
.../server/metrics/RaftServerMetricsImpl.java | 12 +++
.../java/org/apache/ratis/LogAppenderTests.java | 112 ++++++++++++++-------
3 files changed, 104 insertions(+), 39 deletions(-)
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
index 798fb3c4f..b3f872e2b 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/impl/LeaderStateImpl.java
@@ -351,6 +351,8 @@ class LeaderStateImpl implements LeaderState {
private final int stagingCatchupGap;
private final TimeDuration stagingTimeout;
private final RaftServerMetricsImpl raftServerMetrics;
+ // Serialize follower metric registration and updates with cleanup on step-down.
+ private final Object followerMetricsLock = new Object();
private final LogAppenderMetrics logAppenderMetrics;
private final long followerMaxGapThreshold;
private final PendingStepDown pendingStepDown;
@@ -483,7 +485,9 @@ class LeaderStateImpl implements LeaderState {
replyFlusher.stop();
}
logAppenderMetrics.unregister();
- raftServerMetrics.unregister();
+ synchronized (followerMetricsLock) {
+ raftServerMetrics.clearFollowerHeartbeatMetrics();
+ }
pendingRequests.close();
watchRequests.close();
leaderTracer.close();
@@ -687,7 +691,11 @@ class LeaderStateImpl implements LeaderState {
final List<LogAppender> newAppenders = newPeers.stream().map(peer -> {
final FollowerInfo f = new FollowerInfoImpl(server.getMemberId(), peer, this::getPeer, t, nextIndex, caughtUp);
followerInfoMap.put(peer.getId(), f);
- raftServerMetrics.addFollower(peer.getId());
+ synchronized (followerMetricsLock) {
+ if (!isStopped.get()) {
+ raftServerMetrics.addFollower(peer.getId());
+ }
+ }
logAppenderMetrics.addFollowerGauges(peer.getId(), f::getNextIndex, f::getMatchIndex, f::getLastRpcTime);
return newLogAppender(f);
}).collect(Collectors.toList());
@@ -1370,7 +1378,12 @@ class LeaderStateImpl implements LeaderState {
server.getStateMachine().leaderEvent().notifyFollowerSlowness(leaderInfo, follower.getPeer());
}
final RaftPeerId followerId = follower.getId();
- raftServerMetrics.recordFollowerHeartbeatElapsedTime(followerId, elapsedTime.toLong(TimeUnit.NANOSECONDS));
+ synchronized (followerMetricsLock) {
+ // An appender may finish its current iteration after stopAsync() was called.
+ if (!isStopped.get()) {
+ raftServerMetrics.recordFollowerHeartbeatElapsedTime(followerId, elapsedTime.toLong(TimeUnit.NANOSECONDS));
+ }
+ }
}
@Override
diff --git a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java
index 70711c3e3..0b6af9726 100644
--- a/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java
+++ b/ratis-server/src/main/java/org/apache/ratis/server/metrics/RaftServerMetricsImpl.java
@@ -196,6 +196,18 @@ public final class RaftServerMetricsImpl extends RatisMetrics implements RaftSer
addPeerCommitIndexGauge(followerName);
}
+ /**
+ * Remove the heartbeat gauges owned by the leader, retaining server metrics.
+ * Peer commit-index gauges still read the server cache, which is updated as a follower.
+ * The caller must serialize this with follower metric registration and updates.
+ */
+ public void clearFollowerHeartbeatMetrics() {
+ followerLastHeartbeatElapsedTimeMap.keySet().forEach(followerId -> {
+ getRegistry().remove(String.format(FOLLOWER_LAST_HEARTBEAT_ELAPSED_TIME_METRIC, followerId));
+ });
+ followerLastHeartbeatElapsedTimeMap.clear();
+ }
+
/**
* Register a commit index tracker for the peer in cluster.
*/
diff --git a/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java b/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java
index 624ce4757..546f1dafb 100644
--- a/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java
+++ b/ratis-server/src/test/java/org/apache/ratis/LogAppenderTests.java
@@ -18,11 +18,15 @@
package org.apache.ratis;
import static org.apache.ratis.RaftTestUtil.waitForLeader;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.apache.ratis.RaftTestUtil.SimpleMessage;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.metrics.MetricRegistries;
import org.apache.ratis.metrics.impl.RatisMetricRegistryImpl;
import org.apache.ratis.metrics.impl.DefaultTimekeeperImpl;
import org.apache.ratis.proto.RaftProtos.LogEntryProto;
@@ -48,7 +52,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumMap;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.SortedMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -128,50 +134,84 @@ public abstract class LogAppenderTests<CLUSTER extends MiniRaftCluster>
}
@Test
- public void testFollowerHeartbeatMetric() throws IOException, InterruptedException {
-
- // Start a 3 node Ratis ring.
+ public void testFollowerHeartbeatMetric() throws Exception {
final MiniRaftCluster cluster = newCluster(3);
- cluster.start();
- final RaftServer.Division leaderServer = waitForLeader(cluster);
+ final Map<RaftPeerId, RatisMetricRegistryImpl> registries = new HashMap<>();
+ try {
+ cluster.start();
+ final RaftPeerId originalLeader = waitForLeader(cluster).getId();
+ for (RaftServer.Division server : cluster.iterateDivisions()) {
+ registries.put(server.getId(), (RatisMetricRegistryImpl)
+ ((RaftServerMetricsImpl) server.getRaftServerMetrics()).getRegistry());
+ }
- // Write 10 messages to leader.
- try(RaftClient client = cluster.createClient(leaderServer.getId())) {
- for (int i = 1; i <= 10; i++) {
- client.io().send(new RaftTestUtil.SimpleMessage("Msg to make leader ready " + i));
+ // Check initial leadership, step-down, and re-election without restarting the servers.
+ for (int round = 0; round < 3; round++) {
+ final RaftPeerId leaderId = waitForLeader(cluster).getId();
+ try (RaftClient client = cluster.createClient(leaderId)) {
+ for (int i = 0; i < 10; i++) {
+ RaftTestUtil.assertSuccessReply(
+ client.io().send(new SimpleMessage("heartbeat metrics round " + round + " message " + i)));
+ }
+ JavaUtils.attempt(() -> assertFollowerHeartbeatMetrics(cluster, registries),
+ 100, HUNDRED_MILLIS, "check heartbeat metrics", LOG);
+ final RatisMetricRegistryImpl previousLeaderRegistry = registries.get(leaderId);
+ final SortedMap<String, Gauge> commitIndexGauges = previousLeaderRegistry.getGauges(
+ (s, m) -> s.endsWith("_peerCommitIndex"));
+ assertEquals(3, commitIndexGauges.size());
+ if (round < 2) {
+ final RaftPeerId nextLeader = round == 0 ? cluster.getFollowers().get(0).getId() : originalLeader;
+ assertTrue(client.admin().transferLeadership(nextLeader, 20_000).isSuccess());
+ assertEquals(nextLeader, waitForLeader(cluster).getId());
+ // Commit-index gauges read the server cache and remain useful after step-down.
+ final SortedMap<String, Gauge> retainedGauges = previousLeaderRegistry.getGauges(
+ (s, m) -> s.endsWith("_peerCommitIndex"));
+ commitIndexGauges.forEach((name, gauge) -> assertSame(gauge, retainedGauges.get(name)));
+ }
+ }
}
- } catch (IOException e) {
- throw e;
+ } finally {
+ cluster.shutdown();
}
+ registries.values().forEach(registry -> assertTrue(
+ !MetricRegistries.global().get(registry.getMetricRegistryInfo()).isPresent(),
+ "Server registry should be unregistered after shutdown"));
+ }
- final RatisMetricRegistryImpl ratisMetricRegistry = (RatisMetricRegistryImpl)
- ((RaftServerMetricsImpl)leaderServer.getRaftServerMetrics()).getRegistry();
-
- // Get all last_heartbeat_elapsed_time metric gauges. Should be equal to number of followers.
- SortedMap<String, Gauge> heartbeatElapsedTimeGauges = ratisMetricRegistry.getGauges((s, metric) ->
+ private void assertFollowerHeartbeatMetrics(MiniRaftCluster cluster,
+ Map<RaftPeerId, RatisMetricRegistryImpl> registries) {
+ final RaftServer.Division leader = cluster.getLeader();
+ assertNotNull(leader);
+ final RatisMetricRegistryImpl leaderRegistry = registries.get(leader.getId());
+ final SortedMap<String, Gauge> heartbeatGauges = leaderRegistry.getGauges((s, m) ->
s.contains("lastHeartbeatElapsedTime"));
- assertTrue(heartbeatElapsedTimeGauges.size() == 2);
-
- for (RaftServer.Division followerServer : cluster.getFollowers()) {
- String followerId = followerServer.getId().toString();
- Gauge metric = heartbeatElapsedTimeGauges.entrySet().parallelStream().filter(e -> e.getKey().contains(
- followerId)).iterator().next().getValue();
- // Metric for this follower exists.
- assertTrue(metric != null);
- // Metric in nanos > 0.
- assertTrue((long)metric.getValue() > 0);
- // Try to get Heartbeat metrics for follower.
- final RaftServerMetricsImpl followerMetrics = (RaftServerMetricsImpl) followerServer.getRaftServerMetrics();
- // Metric should not exist. It only exists in leader.
- final RatisMetricRegistryImpl followerMetricRegistry = (RatisMetricRegistryImpl)followerMetrics.getRegistry();
- assertTrue(followerMetricRegistry.getGauges((s, m) -> s.contains("lastHeartbeatElapsedTime")).isEmpty());
- for (boolean heartbeat : new boolean[] { true, false }) {
- final DefaultTimekeeperImpl t = (DefaultTimekeeperImpl) followerMetrics.getFollowerAppendEntryTimer(heartbeat);
- assertTrue(t.getTimer().getMeanRate() > 0.0d);
- assertTrue(t.getTimer().getCount() > 0L);
+ assertEquals(2, heartbeatGauges.size());
+
+ for (RaftServer.Division server : cluster.iterateDivisions()) {
+ final RatisMetricRegistryImpl registry = registries.get(server.getId());
+ assertSame(registry, MetricRegistries.global().get(registry.getMetricRegistryInfo())
+ .orElseThrow(() -> new AssertionError("Missing server registry for " + server.getId())));
+ assertTrue(!registry.getGauges((s, m) -> s.endsWith(server.getId() + "_peerCommitIndex")).isEmpty());
+ assertEquals(5, registry.getGauges((s, m) -> s.contains("retryCache")).size());
+
+ if (server.getId().equals(leader.getId())) {
+ continue;
+ }
+ final Gauge<?> heartbeat = heartbeatGauges.entrySet().stream()
+ .filter(e -> e.getKey().endsWith(server.getId() + "_lastHeartbeatElapsedTime"))
+ .findFirst().orElseThrow(() -> new AssertionError("Missing heartbeat gauge for " + server.getId()))
+ .getValue();
+ assertTrue((Long) heartbeat.getValue() > 0);
+ assertTrue(registry.getGauges((s, m) -> s.contains("lastHeartbeatElapsedTime")).isEmpty());
+ assertTrue(registry.getGauges((s, m) -> s.endsWith("numPendingRequestInQueue")
+ || s.endsWith("numPendingRequestMegaByteSize") || s.matches(".*numWatch.*RequestInQueue")).isEmpty());
+ final RaftServerMetricsImpl metrics = (RaftServerMetricsImpl) server.getRaftServerMetrics();
+ for (boolean isHeartbeat : new boolean[] {true, false}) {
+ final DefaultTimekeeperImpl timer = (DefaultTimekeeperImpl) metrics.getFollowerAppendEntryTimer(isHeartbeat);
+ assertTrue(timer.getTimer().getMeanRate() > 0.0d);
+ assertTrue(timer.getTimer().getCount() > 0L);
}
}
- cluster.shutdown();
}
void runTest(CLUSTER cluster) throws Exception {