You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by da...@apache.org on 2022/12/20 15:20:53 UTC

[kafka] branch trunk updated: KAFKA-14531 Fix controller snapshot interval (#13019)

This is an automated email from the ASF dual-hosted git repository.

davidarthur pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 71ea16f8eb2 KAFKA-14531 Fix controller snapshot interval (#13019)
71ea16f8eb2 is described below

commit 71ea16f8eb28e2cc5731a94abfefdb3e57b7aecd
Author: David Arthur <mu...@gmail.com>
AuthorDate: Tue Dec 20 10:20:41 2022 -0500

    KAFKA-14531 Fix controller snapshot interval (#13019)
    
    This patch applies the necessary conversion of milliseconds to nanoseconds for SnapshotGenerator.
    
    Reviewers: Colin P. McCabe <cm...@apache.org>, José Armando García Sancio <js...@gmail.com>
---
 .../src/main/scala/kafka/server/SharedServer.scala |  4 +--
 .../kafka/server/KRaftClusterTest.scala            | 32 ++++++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/kafka/server/SharedServer.scala b/core/src/main/scala/kafka/server/SharedServer.scala
index 151429a1797..6a7cf8a8304 100644
--- a/core/src/main/scala/kafka/server/SharedServer.scala
+++ b/core/src/main/scala/kafka/server/SharedServer.scala
@@ -35,7 +35,7 @@ import org.apache.kafka.server.metrics.KafkaYammerMetrics
 
 import java.util
 import java.util.Collections
-import java.util.concurrent.CompletableFuture
+import java.util.concurrent.{CompletableFuture, TimeUnit}
 import java.util.concurrent.atomic.AtomicReference
 
 
@@ -248,7 +248,7 @@ class SharedServer(
             setTime(time).
             setFaultHandler(metadataPublishingFaultHandler).
             setMaxBytesSinceLastSnapshot(sharedServerConfig.metadataSnapshotMaxNewRecordBytes).
-            setMaxTimeSinceLastSnapshotNs(sharedServerConfig.metadataSnapshotMaxIntervalMs).
+            setMaxTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(sharedServerConfig.metadataSnapshotMaxIntervalMs)).
             setDisabledReason(snapshotsDiabledReason).
             build()
           raftManager.register(loader)
diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
index 987bfea2748..cb91e7cfff5 100644
--- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
+++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
@@ -42,6 +42,8 @@ import org.apache.kafka.server.common.MetadataVersion
 import org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig
 import org.slf4j.LoggerFactory
 
+import java.io.File
+import java.nio.file.{FileSystems, Path}
 import scala.annotation.nowarn
 import scala.collection.mutable
 import scala.concurrent.ExecutionException
@@ -917,4 +919,34 @@ class KRaftClusterTest {
       cluster.close()
     }
   }
+
+  @Test
+  def testSnapshotCount(): Unit = {
+    val cluster = new KafkaClusterTestKit.Builder(
+      new TestKitNodes.Builder().
+        setNumBrokerNodes(0).
+        setNumControllerNodes(1).build())
+      .setConfigProp("metadata.log.max.snapshot.interval.ms", "500")
+      .setConfigProp("metadata.max.idle.interval.ms", "50") // Set this low to generate metadata
+      .build()
+
+    try {
+      cluster.format()
+      cluster.startup()
+      def snapshotCounter(path: Path): Long = {
+       path.toFile.listFiles((_: File, name: String) => {
+          name.toLowerCase.endsWith("checkpoint")
+        }).length
+      }
+
+      val metaLog = FileSystems.getDefault.getPath(cluster.controllers().get(3000).config.metadataLogDir, "__cluster_metadata-0")
+      TestUtils.waitUntilTrue(() => { snapshotCounter(metaLog) > 0 }, "Failed to see at least one snapshot")
+      Thread.sleep(500 * 10) // Sleep for 10 snapshot intervals
+      val countAfterTenIntervals = snapshotCounter(metaLog)
+      assertTrue(countAfterTenIntervals > 1, s"Expected to see at least one more snapshot, saw $countAfterTenIntervals")
+      assertTrue(countAfterTenIntervals < 20, s"Did not expect to see more than twice as many snapshots as snapshot intervals, saw $countAfterTenIntervals")
+    } finally {
+      cluster.close()
+    }
+  }
 }