You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ij...@apache.org on 2025/01/19 15:21:58 UTC

(kafka) branch trunk updated: KAFKA-17668: Clean-up LogCleaner#maxOverCleanerThreads and LogCleanerManager#maintainUncleanablePartitions (#17390)

This is an automated email from the ASF dual-hosted git repository.

ijuma pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 697cf641bca KAFKA-17668: Clean-up LogCleaner#maxOverCleanerThreads and LogCleanerManager#maintainUncleanablePartitions (#17390)
697cf641bca is described below

commit 697cf641bca8036c5dd4cb99aa87a98a6e0443d0
Author: TengYao Chi <ki...@gmail.com>
AuthorDate: Sun Jan 19 23:21:52 2025 +0800

    KAFKA-17668: Clean-up LogCleaner#maxOverCleanerThreads and LogCleanerManager#maintainUncleanablePartitions (#17390)
    
    Since maxOverCleanerThreads does not have a corresponding unit test,
    I have added a unit test for it. maintainUncleanablePartitions has been
    thoroughly tested in tests, so I simply replaced the old implementation
    with the new one.
    
    Reviewers: Ismael Juma <is...@juma.me.uk>, Chia-Ping Tsai <ch...@gmail.com>
---
 core/src/main/scala/kafka/log/LogCleaner.scala     |  5 ++-
 .../main/scala/kafka/log/LogCleanerManager.scala   | 16 +++------
 .../test/scala/unit/kafka/log/LogCleanerTest.scala | 39 ++++++++++++++++++++++
 3 files changed, 46 insertions(+), 14 deletions(-)

diff --git a/core/src/main/scala/kafka/log/LogCleaner.scala b/core/src/main/scala/kafka/log/LogCleaner.scala
index 71747e69dc2..c07437e68d5 100644
--- a/core/src/main/scala/kafka/log/LogCleaner.scala
+++ b/core/src/main/scala/kafka/log/LogCleaner.scala
@@ -115,12 +115,11 @@ class LogCleaner(initialConfig: CleanerConfig,
   private[log] val cleaners = mutable.ArrayBuffer[CleanerThread]()
 
   /**
-   * scala 2.12 does not support maxOption so we handle the empty manually.
    * @param f to compute the result
    * @return the max value (int value) or 0 if there is no cleaner
    */
-  private def maxOverCleanerThreads(f: CleanerThread => Double): Int =
-    cleaners.foldLeft(0.0d)((max: Double, thread: CleanerThread) => math.max(max, f(thread))).toInt
+  private[log] def maxOverCleanerThreads(f: CleanerThread => Double): Int =
+    cleaners.map(f).maxOption.getOrElse(0.0d).toInt
 
   /* a metric to track the maximum utilization of any thread's buffer in the last cleaning */
   metricsGroup.newGauge(MaxBufferUtilizationPercentMetricName,
diff --git a/core/src/main/scala/kafka/log/LogCleanerManager.scala b/core/src/main/scala/kafka/log/LogCleanerManager.scala
index 3e126e45ffe..983e87dd437 100755
--- a/core/src/main/scala/kafka/log/LogCleanerManager.scala
+++ b/core/src/main/scala/kafka/log/LogCleanerManager.scala
@@ -532,21 +532,15 @@ private[log] class LogCleanerManager(val logDirs: Seq[File],
   def maintainUncleanablePartitions(): Unit = {
     // Remove deleted partitions from uncleanablePartitions
     inLock(lock) {
-      // Note: we don't use retain or filterInPlace method in this function because retain is deprecated in
-      // scala 2.13 while filterInPlace is not available in scala 2.12.
-
       // Remove deleted partitions
-      uncleanablePartitions.values.foreach {
-        partitions =>
-          val partitionsToRemove = partitions.filterNot(logs.contains).toList
-          partitionsToRemove.foreach { partitions.remove }
+      uncleanablePartitions.values.foreach { partitions =>
+        partitions.filterInPlace(logs.contains)
       }
 
       // Remove entries with empty partition set.
-      val logDirsToRemove = uncleanablePartitions.filter {
-        case (_, partitions) => partitions.isEmpty
-      }.keys.toList
-      logDirsToRemove.foreach { uncleanablePartitions.remove }
+      uncleanablePartitions.filterInPlace {
+        case (_, partitions) => partitions.nonEmpty
+      }
     }
   }
 
diff --git a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
index f7746c7a69b..b83a36a4b5d 100644
--- a/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
+++ b/core/src/test/scala/unit/kafka/log/LogCleanerTest.scala
@@ -2045,6 +2045,45 @@ class LogCleanerTest extends Logging {
     }
   }
 
+  @Test
+  def testMaxOverCleanerThreads(): Unit = {
+    val logCleaner =  new LogCleaner(new CleanerConfig(true),
+      logDirs = Array(TestUtils.tempDir(), TestUtils.tempDir()),
+      logs = new Pool[TopicPartition, UnifiedLog](),
+      logDirFailureChannel = new LogDirFailureChannel(1),
+      time = time)
+
+    val cleaners = logCleaner.cleaners
+
+    val cleaner1 = new logCleaner.CleanerThread(1)
+    cleaner1.lastStats = new CleanerStats(time)
+    cleaner1.lastStats.bufferUtilization = 0.75
+    cleaners += cleaner1
+
+    val cleaner2 = new logCleaner.CleanerThread(2)
+    cleaner2.lastStats = new CleanerStats(time)
+    cleaner2.lastStats.bufferUtilization = 0.85
+    cleaners += cleaner2
+
+    val cleaner3 = new logCleaner.CleanerThread(3)
+    cleaner3.lastStats = new CleanerStats(time)
+    cleaner3.lastStats.bufferUtilization = 0.65
+    cleaners += cleaner3
+
+    assertEquals(0, logCleaner.maxOverCleanerThreads(_.lastStats.bufferUtilization))
+
+    cleaners.clear()
+
+    cleaner1.lastStats.bufferUtilization = 5d
+    cleaners += cleaner1
+    cleaner2.lastStats.bufferUtilization = 6d
+    cleaners += cleaner2
+    cleaner3.lastStats.bufferUtilization = 7d
+    cleaners += cleaner3
+
+    assertEquals(7, logCleaner.maxOverCleanerThreads(_.lastStats.bufferUtilization))
+  }
+
   private def writeToLog(log: UnifiedLog, keysAndValues: Iterable[(Int, Int)], offsetSeq: Iterable[Long]): Iterable[Long] = {
     for (((key, value), offset) <- keysAndValues.zip(offsetSeq))
       yield log.appendAsFollower(messageWithOffset(key, value, offset)).lastOffset