You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by sh...@apache.org on 2023/12/21 07:45:31 UTC

(kafka) branch trunk updated: KAFKA-16013: Throw an exception in DelayedRemoteFetch for follower fetch replicas. (#15015)

This is an automated email from the ASF dual-hosted git repository.

showuon pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new d59d6132587 KAFKA-16013: Throw an exception in DelayedRemoteFetch for follower fetch replicas. (#15015)
d59d6132587 is described below

commit d59d61325872d256474ba30ea858a0f3877c8331
Author: Luke Chen <sh...@gmail.com>
AuthorDate: Thu Dec 21 15:45:24 2023 +0800

    KAFKA-16013: Throw an exception in DelayedRemoteFetch for follower fetch replicas. (#15015)
    
    Follow-up for KAFKA-16013: Add metric for expiration rate of delayed remote fetch
    
    Reviewers: Nikhil Ramakrishnan <ra...@gmail.com>, Kamal Chandraprakash <ka...@gmail.com>,  Satish Duggana <sa...@apache.org>
---
 .../scala/kafka/server/DelayedRemoteFetch.scala    |  9 ++++---
 .../kafka/log/remote/RemoteLogManagerTest.java     |  4 ---
 .../kafka/server/DelayedRemoteFetchTest.scala      | 31 +++++++++++++++++-----
 .../unit/kafka/server/ReplicaManagerTest.scala     | 27 ++++++++++++-------
 4 files changed, 48 insertions(+), 23 deletions(-)

diff --git a/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala b/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala
index 143201f3d83..b74886f574f 100644
--- a/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala
+++ b/core/src/main/scala/kafka/server/DelayedRemoteFetch.scala
@@ -41,6 +41,10 @@ class DelayedRemoteFetch(remoteFetchTask: Future[Void],
                          responseCallback: Seq[(TopicIdPartition, FetchPartitionData)] => Unit)
   extends DelayedOperation(fetchParams.maxWaitMs) {
 
+  if (fetchParams.isFromFollower) {
+    throw new IllegalStateException(s"The follower should not invoke remote fetch. Fetch params are: $fetchParams")
+  }
+
   /**
    * The operation can be completed if:
    *
@@ -82,10 +86,7 @@ class DelayedRemoteFetch(remoteFetchTask: Future[Void],
     val cancelled = remoteFetchTask.cancel(true)
     if (!cancelled) debug(s"Remote fetch task for RemoteStorageFetchInfo: $remoteFetchInfo could not be cancelled and its isDone value is ${remoteFetchTask.isDone}")
 
-    if (fetchParams.isFromFollower)
-      warn(s"The follower should not invoke remote fetch. Fetch params are: $fetchParams")
-    else
-      DelayedRemoteFetchMetrics.expiredRequestMeter.mark()
+    DelayedRemoteFetchMetrics.expiredRequestMeter.mark()
   }
 
   /**
diff --git a/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java b/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java
index d581d64d7c8..c3e3c4300a9 100644
--- a/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java
+++ b/core/src/test/java/kafka/log/remote/RemoteLogManagerTest.java
@@ -846,10 +846,6 @@ public class RemoteLogManagerTest {
                 String.format("Expected to find 1 for RemoteCopyLagSegments metric value, but found %d", safeLongYammerMetricValue("RemoteCopyLagSegments")));
         // unlock copyLogSegmentData
         latch.countDown();
-
-        TestUtils.waitForCondition(
-                () -> safeLongYammerMetricValue("RemoteLogSizeComputationTime") >= 1000,
-                String.format("Expected to find 1000 for RemoteLogSizeComputationTime metric value, but found %d", safeLongYammerMetricValue("RemoteLogSizeComputationTime")));
     }
 
     private Object yammerMetricValue(String name) {
diff --git a/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala b/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala
index c943258da8f..c35385bcbc8 100644
--- a/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala
+++ b/core/src/test/scala/integration/kafka/server/DelayedRemoteFetchTest.scala
@@ -40,12 +40,11 @@ class DelayedRemoteFetchTest {
   private val fetchOffset = 500L
   private val logStartOffset = 0L
   private val currentLeaderEpoch = Optional.of[Integer](10)
-  private val replicaId = 1
 
   private val fetchStatus = FetchPartitionStatus(
     startOffsetMetadata = new LogOffsetMetadata(fetchOffset),
     fetchInfo = new FetchRequest.PartitionData(Uuid.ZERO_UUID, fetchOffset, logStartOffset, maxBytes, currentLeaderEpoch))
-  private val fetchParams = buildFollowerFetchParams(replicaId, maxWaitMs = 500)
+  private val fetchParams = buildFetchParams(replicaId = -1, maxWaitMs = 500)
 
   @Test
   def testFetch(): Unit = {
@@ -83,6 +82,28 @@ class DelayedRemoteFetchTest {
     assertEquals(leaderLogStartOffset, fetchResult.logStartOffset)
   }
 
+  @Test
+  def testFollowerFetch(): Unit = {
+    var actualTopicPartition: Option[TopicIdPartition] = None
+    var fetchResultOpt: Option[FetchPartitionData] = None
+
+    def callback(responses: Seq[(TopicIdPartition, FetchPartitionData)]): Unit = {
+      assertEquals(1, responses.size)
+      actualTopicPartition = Some(responses.head._1)
+      fetchResultOpt = Some(responses.head._2)
+    }
+
+    val future: CompletableFuture[RemoteLogReadResult] = new CompletableFuture[RemoteLogReadResult]()
+    future.complete(null)
+    val fetchInfo: RemoteStorageFetchInfo = new RemoteStorageFetchInfo(0, false, topicIdPartition.topicPartition(), null, null, false)
+    val highWatermark = 100
+    val leaderLogStartOffset = 10
+    val logReadInfo = buildReadResult(Errors.NONE, highWatermark, leaderLogStartOffset)
+    val fetchParams = buildFetchParams(replicaId = 1, maxWaitMs = 500)
+    assertThrows(classOf[IllegalStateException], () => new DelayedRemoteFetch(null, future, fetchInfo, Seq(topicIdPartition -> fetchStatus), fetchParams,
+      Seq(topicIdPartition -> logReadInfo), replicaManager, callback))
+  }
+
   @Test
   def testNotLeaderOrFollower(): Unit = {
     var actualTopicPartition: Option[TopicIdPartition] = None
@@ -155,14 +176,12 @@ class DelayedRemoteFetchTest {
       fetchResultOpt = Some(responses.head._2)
     }
 
-    val replicaId = -1
     val highWatermark = 100
     val leaderLogStartOffset = 10
 
     val remoteFetchTask = mock(classOf[Future[Void]])
     val future: CompletableFuture[RemoteLogReadResult] = new CompletableFuture[RemoteLogReadResult]()
     val fetchInfo: RemoteStorageFetchInfo = new RemoteStorageFetchInfo(0, false, topicIdPartition.topicPartition(), null, null, false)
-    val fetchParams = buildFollowerFetchParams(replicaId, maxWaitMs = 500)
     val logReadInfo = buildReadResult(Errors.NONE, highWatermark, leaderLogStartOffset)
 
     val delayedRemoteFetch = new DelayedRemoteFetch(remoteFetchTask, future, fetchInfo, Seq(topicIdPartition -> fetchStatus), fetchParams,
@@ -196,8 +215,8 @@ class DelayedRemoteFetchTest {
     assertEquals(leaderLogStartOffset, fetchResult.logStartOffset)
   }
 
-  private def buildFollowerFetchParams(replicaId: Int,
-                                       maxWaitMs: Int): FetchParams = {
+  private def buildFetchParams(replicaId: Int,
+                               maxWaitMs: Int): FetchParams = {
     new FetchParams(
       ApiKeys.FETCH.latestVersion,
       replicaId,
diff --git a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
index 78179299934..0cb14b543df 100644
--- a/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
+++ b/core/src/test/scala/unit/kafka/server/ReplicaManagerTest.scala
@@ -67,7 +67,7 @@ import org.junit.jupiter.api.Assertions._
 import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.{EnumSource, ValueSource}
-import com.yammer.metrics.core.Gauge
+import com.yammer.metrics.core.{Gauge, Meter}
 import kafka.log.remote.RemoteLogManager
 import org.apache.kafka.common.config.{AbstractConfig, TopicConfig}
 import org.apache.kafka.metadata.properties.{MetaProperties, MetaPropertiesEnsemble, MetaPropertiesVersion, PropertiesUtils}
@@ -4019,6 +4019,7 @@ class ReplicaManagerTest {
       .getOrElse(fail(s"Unable to find broker metric $name: allMetrics: ${allMetrics.keySet.map(_.getMBeanName)}"))
     metric match {
       case m: Gauge[_] => m.value
+      case m: Meter => m.count()
       case m => fail(s"Unexpected broker metric of class ${m.getClass}")
     }
   }
@@ -4122,11 +4123,15 @@ class ReplicaManagerTest {
       // Replicas fetch from the leader periodically, therefore we check that the metric value is increasing
       // We expect failedBuildRemoteLogAuxStateRate to increase because there is no remoteLogSegmentMetadata
       // when attempting to build log aux state
-      assertTrue(brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0)
-      assertTrue(brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0,
+        "Should have buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0,
+        "Should have failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count)
       // Verify aggregate metrics
-      assertTrue(brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0)
-      assertTrue( brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0,
+        "Should have all topic buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0,
+        "Should have all topic failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count)
     } finally {
       replicaManager.shutdown(checkpointHW = false)
     }
@@ -4176,11 +4181,15 @@ class ReplicaManagerTest {
 
       // Replicas fetch from the leader periodically, therefore we check that the metric value is increasing
       // We expect failedBuildRemoteLogAuxStateRate to increase because fetchRemoteLogSegmentMetadata returns RemoteStorageException
-      assertTrue(brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0)
-      assertTrue(brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count > 0,
+        "Should have buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).buildRemoteLogAuxStateRequestRate.count)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count > 0,
+        "Should have failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.topicStats(tp0.topic()).failedBuildRemoteLogAuxStateRate.count)
       // Verify aggregate metrics
-      assertTrue(brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0)
-      assertTrue(brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count > 0,
+        "Should have all topic buildRemoteLogAuxStateRequestRate count > 0, but got:" + brokerTopicStats.allTopicsStats.buildRemoteLogAuxStateRequestRate.count)
+      TestUtils.waitUntilTrue(() => brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count > 0,
+        "Should have all topic failedBuildRemoteLogAuxStateRate count > 0, but got:" + brokerTopicStats.allTopicsStats.failedBuildRemoteLogAuxStateRate.count)
     } finally {
       replicaManager.shutdown(checkpointHW = false)
     }