You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by sh...@apache.org on 2023/12/21 07:45:31 UTC
(kafka) branch trunk updated: KAFKA-16013: Throw an exception in DelayedRemoteFetch for follower fetch replicas. (#15015)
This is an automated email from the ASF dual-hosted git repository.
showuon pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new d59d6132587 KAFKA-16013: Throw an exception in DelayedRemoteFetch for follower fetch replicas. (#15015)
d59d6132587 is described below
commit d59d61325872d256474ba30ea858a0f3877c8331
Author: Luke Chen <sh...@gmail.com>
AuthorDate: Thu Dec 21 15:45:24 2023 +0800
KAFKA-16013: Throw an exception in DelayedRemoteFetch for follower fetch replicas. (#15015)
Follow-up for KAFKA-16013: Add metric for expiration rate of delayed remote fetch
Reviewers: Nikhil Ramakrishnan <ra...@gmail.com>, Kamal Chandraprakash <ka...@gmail.com>, Satish Duggana <sa...@apache.org>
---
.../scala/kafka/server/DelayedRemoteFetch.scala | 9 ++++---
.../kafka/log/remote/RemoteLogManagerTest.java | 4 ---
.../kafka/server/DelayedRemoteFetchTest.scala | 31 +++++++++++++++++-----
.../unit/kafka/server/ReplicaManagerTest.scala | 27 ++++++++++++-------
4 files changed, 48 insertions(+), 23 deletions(-)
diff --git a/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala b/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala
index 143201f3d83..b74886f574f 100644
--- a/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala
+++ b/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala
@@ -41,6 +41,10 @@ class DelayedRemoteFetch(remoteFetchTask: Future[Void],
responseCallback: Seq[(TopicIdPartition, FetchPartitionData)] => Unit)
extends DelayedOperation(fetchParams.maxWaitMs) {
+ if (fetchParams.isFromFollower) {
+ throw new IllegalStateException(s"The follower should not invoke remote fetch. Fetch params are: $fetchParams")
+ }
+
/**
* The operation can be completed if:
*
@@ -82,10 +86,7 @@ class DelayedRemoteFetch(remoteFetchTask: Future[Void],
val cancelled = remoteFetchTask.cancel(true)
if (!cancelled) debug(s"Remote fetch task for RemoteStorageFetchInfo: $remoteFetchInfo could not be cancelled and its isDone value is ${remoteFetchTask.isDone}")
- if (fetchParams.isFromFollower)
- warn(s"The follower should not invoke remote fetch. Fetch params are: $fetchParams")
- else
- DelayedRemoteFetchMetrics.expiredRequestMeter.mark()
+ DelayedRemoteFetchMetrics.expiredRequestMeter.mark()
}
/**
diff --git a/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java b/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java
index d581d64d7c8..c3e3c4300a9 100644
--- a/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java
+++ b/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java
@@ -846,10 +846,6 @@ public class RemoteLogManagerTest {
String.format("Expected to find 1 for RemoteCopyLagSegments metric value, but found %d", safeLongYammerMetricValue("RemoteCopyLagSegments")));
// unlock copyLogSegmentData
latch.countDown();
-
- TestUtils.waitForCondition(
- () -> safeLongYammerMetricValue("RemoteLogSizeComputationTime") >= 1000,
- String.format("Expected to find 1000 for RemoteLogSizeComputationTime metric value, but found %d", safeLongYammerMetricValue("RemoteLogSizeComputationTime")));
}
private Object yammerMetricValue(String name) {
diff --git a/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala b/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala
index c943258da8f..c35385bcbc8 100644
--- a/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala
@@ -40,12 +40,11 @@ class DelayedRemoteFetchTest {
private val fetchOffset = 500L
private val logStartOffset = 0L
private val currentLeaderEpoch = Optional.of[Integer](10)
- private val replicaId = 1
private val fetchStatus = FetchPartitionStatus(
startOffsetMetadata = new LogOffsetMetadata(fetchOffset),
fetchInfo = new FetchRequest.PartitionData(Uuid.ZERO_UUID, fetchOffset, logStartOffset, maxBytes, currentLeaderEpoch))
- private val fetchParams = buildFollowerFetchParams(replicaId, maxWaitMs = 500)
+ private val fetchParams = buildFetchParams(replicaId = -1, maxWaitMs = 500)
@Test
def testFetch(): Unit = {
@@ -83,6 +82,28 @@ class DelayedRemoteFetchTest {
assertEquals(leaderLogStartOffset, fetchResult.logStartOffset)
}
+ @Test
+ def testFollowerFetch(): Unit = {
+ var actualTopicPartition: Option[TopicIdPartition] = None
+ var fetchResultOpt: Option[FetchPartitionData] = None
+
+ def callback(responses: Seq[(TopicIdPartition, FetchPartitionData)]): Unit = {
+ assertEquals(1, responses.size)
+ actualTopicPartition = Some(responses.head._1)
+ fetchResultOpt = Some(responses.head._2)
+ }
+
+ val future: CompletableFuture[RemoteLogReadResult] = new CompletableFuture[RemoteLogReadResult]()
+ future.complete(null)
+ val fetchInfo: RemoteStorageFetchInfo = new RemoteStorageFetchInfo(0, false, topicIdPartition.topicPartition(), null, null, false)
+ val highWatermark = 100
+ val leaderLogStartOffset = 10
+ val logReadInfo = buildReadResult(Errors.NONE, highWatermark, leaderLogStartOffset)
+ val fetchParams = buildFetchParams(replicaId = 1, maxWaitMs = 500)
+ assertThrows(classOf[IllegalStateException], () => new DelayedRemoteFetch(null, future, fetchInfo, Seq(topicIdPartition -> fetchStatus), fetchParams,
+ Seq(topicIdPartition -> logReadInfo), replicaManager, callback))
+ }
+
@Test
def testNotLeaderOrFollower(): Unit = {
var actualTopicPartition: Option[TopicIdPartition] = None
@@ -155,14 +176,12 @@ class DelayedRemoteFetchTest {
fetchResultOpt = Some(responses.head._2)
}
- val replicaId = -1
val highWatermark = 100
val leaderLogStartOffset = 10
val remoteFetchTask = mock(classOf[Future[Void]])
val future: CompletableFuture[RemoteLogReadResult] = new CompletableFuture[RemoteLogReadResult]()
val fetchInfo: RemoteStorageFetchInfo = new RemoteStorageFetchInfo(0, false, topicIdPartition.topicPartition(), null, null, false)
- val fetchParams = buildFollowerFetchParams(replicaId, maxWaitMs = 500)
val logReadInfo = buildReadResult(Errors.NONE, highWatermark, leaderLogStartOffset)
val delayedRemoteFetch = new DelayedRemoteFetch(remoteFetchTask, future, fetchInfo, Seq(topicIdPartition -> fetchStatus), fetchParams,
@@ -196,8 +215,8 @@ class DelayedRemoteFetchTest {
assertEquals(leaderLogStartOffset, fetchResult.logStartOffset)
}
- private def buildFollowerFetchParams(replicaId: Int,
- maxWaitMs: Int): FetchParams = {
+ private def buildFetchParams(replicaId: Int,
+ maxWaitMs: Int): FetchParams = {
new FetchParams(
ApiKeys.FETCH.latestVersion,
replicaId,
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 78179299934..0cb14b543df 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -67,7 +67,7 @@ import org.junit.jupiter.api.Assertions._
import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.{EnumSource, ValueSource}
-import com.yammer.metrics.core.Gauge
+import com.yammer.metrics.core.{Gauge, Meter}
import kafka.log.remote.RemoteLogManager
import org.apache.kafka.common.config.{AbstractConfig, TopicConfig}
import org.apache.kafka.metadata.properties.{MetaProperties, MetaPropertiesEnsemble, MetaPropertiesVersion, PropertiesUtils}
@@ -4019,6 +4019,7 @@ class ReplicaManagerTest {
.getOrElse(fail(s"Unable to find broker metric $name: allMetrics: ${allMetrics.keySet.map(_.getMBeanName)}"))
metric match {
case m: Gauge[_] => m.value
+ case m: Meter => m.count()
case m => fail(s"Unexpected broker metric of class ${m.getClass}")
}
}
@@ -4122,11 +4123,15 @@ class ReplicaManagerTest {
// Replicas fetch from the leader periodically, therefore we check that the metric value is increasing
// We expect failedBuildRemoteLogAuxStateRate to increase because there is no remoteLogSegmentMetadata
// when attempting to build log aux state
- assertTrue(brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0)
- assertTrue(brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0,
+ "Should have buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0,
+ "Should have failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count)
// Verify aggregate metrics
- assertTrue(brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0)
- assertTrue( brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0,
+ "Should have all topic buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0,
+ "Should have all topic failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count)
} finally {
replicaManager.shutdown(checkpointHW = false)
}
@@ -4176,11 +4181,15 @@ class ReplicaManagerTest {
// Replicas fetch from the leader periodically, therefore we check that the metric value is increasing
// We expect failedBuildRemoteLogAuxStateRate to increase because fetchRemoteLogSegmentMetadata returns RemoteStorageException
- assertTrue(brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0)
- assertTrue(brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0,
+ "Should have buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0,
+ "Should have failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count)
// Verify aggregate metrics
- assertTrue(brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0)
- assertTrue(brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0,
+ "Should have all topic buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count)
+ TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0,
+ "Should have all topic failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count)
} finally {
replicaManager.shutdown(checkpointHW = false)
}