You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by da...@apache.org on 2022/12/20 15:20:53 UTC
[kafka] branch trunk updated: KAFKA-14531 Fix controller snapshot interval (#13019)
This is an automated email from the ASF dual-hosted git repository.
davidarthur pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 71ea16f8eb2 KAFKA-14531 Fix controller snapshot interval (#13019)
71ea16f8eb2 is described below
commit 71ea16f8eb28e2cc5731a94abfefdb3e57b7aecd
Author: David Arthur <mu...@gmail.com>
AuthorDate: Tue Dec 20 10:20:41 2022 -0500
KAFKA-14531 Fix controller snapshot interval (#13019)
This patch applies the necessary conversion of milliseconds to nanoseconds for SnapshotGenerator.
Reviewers: Colin P. McCabe <cm...@apache.org>, José Armando García Sancio <js...@gmail.com>
---
.../src/main/scala/kafka/server/SharedServer.scala | 4 +--
.../kafka/server/KRaftClusterTest.scala | 32 ++++++++++++++++++++++
2 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/kafka/server/SharedServer.scala b/core/src/main/scala/kafka/server/SharedServer.scala
index 151429a1797..6a7cf8a8304 100644
--- a/core/src/main/scala/kafka/server/SharedServer.scala
+++ b/core/src/main/scala/kafka/server/SharedServer.scala
@@ -35,7 +35,7 @@ import org.apache.kafka.server.metrics.KafkaYammerMetrics
import java.util
import java.util.Collections
-import java.util.concurrent.CompletableFuture
+import java.util.concurrent.{CompletableFuture, TimeUnit}
import java.util.concurrent.atomic.AtomicReference
@@ -248,7 +248,7 @@ class SharedServer(
setTime(time).
setFaultHandler(metadataPublishingFaultHandler).
setMaxBytesSinceLastSnapshot(sharedServerConfig.metadataSnapshotMaxNewRecordBytes).
- setMaxTimeSinceLastSnapshotNs(sharedServerConfig.metadataSnapshotMaxIntervalMs).
+ setMaxTimeSinceLastSnapshotNs(TimeUnit.MILLISECONDS.toNanos(sharedServerConfig.metadataSnapshotMaxIntervalMs)).
setDisabledReason(snapshotsDiabledReason).
build()
raftManager.register(loader)
diff --git a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
index 987bfea2748..cb91e7cfff5 100644
--- a/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
+++ b/core/src/test/scala/integration/kafka/server/KRaftClusterTest.scala
@@ -42,6 +42,8 @@ import org.apache.kafka.server.common.MetadataVersion
import org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig
import org.slf4j.LoggerFactory
+import java.io.File
+import java.nio.file.{FileSystems, Path}
import scala.annotation.nowarn
import scala.collection.mutable
import scala.concurrent.ExecutionException
@@ -917,4 +919,34 @@ class KRaftClusterTest {
cluster.close()
}
}
+
+ @Test
+ def testSnapshotCount(): Unit = {
+ val cluster = new KafkaClusterTestKit.Builder(
+ new TestKitNodes.Builder().
+ setNumBrokerNodes(0).
+ setNumControllerNodes(1).build())
+ .setConfigProp("metadata.log.max.snapshot.interval.ms", "500")
+ .setConfigProp("metadata.max.idle.interval.ms", "50") // Set this low to generate metadata
+ .build()
+
+ try {
+ cluster.format()
+ cluster.startup()
+ def snapshotCounter(path: Path): Long = {
+ path.toFile.listFiles((_: File, name: String) => {
+ name.toLowerCase.endsWith("checkpoint")
+ }).length
+ }
+
+ val metaLog = FileSystems.getDefault.getPath(cluster.controllers().get(3000).config.metadataLogDir, "__cluster_metadata-0")
+ TestUtils.waitUntilTrue(() => { snapshotCounter(metaLog) > 0 }, "Failed to see at least one snapshot")
+ Thread.sleep(500 * 10) // Sleep for 10 snapshot intervals
+ val countAfterTenIntervals = snapshotCounter(metaLog)
+ assertTrue(countAfterTenIntervals > 1, s"Expected to see at least one more snapshot, saw $countAfterTenIntervals")
+ assertTrue(countAfterTenIntervals < 20, s"Did not expect to see more than twice as many snapshots as snapshot intervals, saw $countAfterTenIntervals")
+ } finally {
+ cluster.close()
+ }
+ }
}