You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by jg...@apache.org on 2017/01/27 05:27:48 UTC

kafka git commit: MINOR: Close create topics policy during shutdown and more tests

Repository: kafka
Updated Branches:
  refs/heads/trunk a01b72236 -> 2f9e9195a


MINOR: Close create topics policy during shutdown and more tests

Author: Ismael Juma <is...@juma.me.uk>

Reviewers: Jason Gustafson <ja...@confluent.io>

Closes #2443 from ijuma/close-create-topics-policy-during-shutdown


Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/2f9e9195
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/2f9e9195
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/2f9e9195

Branch: refs/heads/trunk
Commit: 2f9e9195abada7c2bcbd41147cb9d4c00f3a712d
Parents: a01b722
Author: Ismael Juma <is...@juma.me.uk>
Authored: Thu Jan 26 21:27:44 2017 -0800
Committer: Jason Gustafson <ja...@confluent.io>
Committed: Thu Jan 26 21:27:44 2017 -0800

----------------------------------------------------------------------
 .../main/scala/kafka/server/AdminManager.scala  |  1 +
 .../CreateTopicsRequestWithPolicyTest.scala     | 49 ++++++++++++++++----
 2 files changed, 42 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kafka/blob/2f9e9195/core/src/main/scala/kafka/server/AdminManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 6dc224d..98a4b18 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -191,5 +191,6 @@ class AdminManager(val config: KafkaConfig,
 
   def shutdown() {
     topicPurgatory.shutdown()
+    CoreUtils.swallow(createTopicPolicy.foreach(_.close()))
   }
 }

http://git-wip-us.apache.org/repos/asf/kafka/blob/2f9e9195/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
index 9affea4..f2020cf 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
@@ -20,6 +20,7 @@ package kafka.server
 import java.util
 import java.util.Properties
 
+import kafka.log.LogConfig
 import kafka.utils.TestUtils
 import org.apache.kafka.common.errors.PolicyViolationException
 import org.apache.kafka.common.protocol.Errors
@@ -48,9 +49,13 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
     validateValidCreateTopicsRequests(new CreateTopicsRequest.Builder(
       Map("topic2" -> new CreateTopicsRequest.TopicDetails(5, 3.toShort)).asJava, timeout, true).build())
 
+    val configs = Map(LogConfig.RetentionMsProp -> 4999.toString)
+    validateValidCreateTopicsRequests(new CreateTopicsRequest.Builder(
+      Map("topic3" -> new CreateTopicsRequest.TopicDetails(11, 2.toShort, configs.asJava)).asJava, timeout, true).build())
+
     val assignments = replicaAssignmentToJava(Map(0 -> List(1, 0), 1 -> List(0, 1)))
     validateValidCreateTopicsRequests(new CreateTopicsRequest.Builder(
-      Map("topic3" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build())
+      Map("topic4" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build())
   }
 
   @Test
@@ -65,14 +70,23 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
       Map("policy-topic1" -> error(Errors.POLICY_VIOLATION, Some("Topics should have at least 5 partitions, received 4"))))
 
     validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
-      Map("policy-topic2" -> new CreateTopicsRequest.TopicDetails(4, 1.toShort)).asJava, timeout, true).build(),
+      Map("policy-topic2" -> new CreateTopicsRequest.TopicDetails(4, 3.toShort)).asJava, timeout, true).build(),
       Map("policy-topic2" -> error(Errors.POLICY_VIOLATION, Some("Topics should have at least 5 partitions, received 4"))))
 
+    val configs = Map(LogConfig.RetentionMsProp -> 5001.toString)
+    validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
+      Map("policy-topic3" -> new CreateTopicsRequest.TopicDetails(11, 2.toShort, configs.asJava)).asJava, timeout, true).build(),
+      Map("policy-topic3" -> error(Errors.POLICY_VIOLATION, Some("RetentionMs should be less than 5000ms if replicationFactor > 5"))))
+
+    validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
+      Map("policy-topic4" -> new CreateTopicsRequest.TopicDetails(11, 3.toShort, Map.empty.asJava)).asJava, timeout, true).build(),
+      Map("policy-topic4" -> error(Errors.POLICY_VIOLATION, Some("RetentionMs should be less than 5000ms if replicationFactor > 5"))))
+
     val assignments = replicaAssignmentToJava(Map(0 -> List(1), 1 -> List(0)))
     validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
-      Map("policy-topic3" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build(),
-      Map("policy-topic3" -> error(Errors.POLICY_VIOLATION,
-        Some("""Topic partitions should have at least 2 partitions, received 1 for partition 0"""))), checkErrorMessage = true)
+      Map("policy-topic5" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build(),
+      Map("policy-topic5" -> error(Errors.POLICY_VIOLATION,
+        Some("Topic partitions should have at least 2 partitions, received 1 for partition 0"))))
 
     // Check that basic errors still work
     validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
@@ -94,29 +108,48 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
 object CreateTopicsRequestWithPolicyTest {
 
   class Policy extends CreateTopicPolicy {
-    def configure(configs: util.Map[String, _]): Unit = ()
+
+    var configs: Map[String, _] = _
+    var closed = false
+
+    def configure(configs: util.Map[String, _]): Unit = {
+      this.configs = configs.asScala.toMap
+    }
 
     def validate(requestMetadata: RequestMetadata): Unit = {
+      require(!closed, "Policy should not be closed")
+      require(!configs.isEmpty, "configure should have been called with non empty configs")
+
       import requestMetadata._
-      require(configs.isEmpty, s"Topic configs should be empty, but it is $configs")
       if (numPartitions != null || replicationFactor != null) {
         require(numPartitions != null, s"numPartitions should not be null, but it is $numPartitions")
         require(replicationFactor != null, s"replicationFactor should not be null, but it is $replicationFactor")
         require(replicasAssignments == null, s"replicaAssigments should be null, but it is $replicasAssignments")
+
         if (numPartitions < 5)
           throw new PolicyViolationException(s"Topics should have at least 5 partitions, received $numPartitions")
+
+        if (numPartitions > 10) {
+          if (requestMetadata.configs.asScala.get(LogConfig.RetentionMsProp).fold(true)(_.toInt > 5000))
+            throw new PolicyViolationException("RetentionMs should be less than 5000ms if replicationFactor > 5")
+        } else
+          require(requestMetadata.configs.isEmpty, s"Topic configs should be empty, but it is ${requestMetadata.configs}")
+
       } else {
         require(numPartitions == null, s"numPartitions should be null, but it is $numPartitions")
         require(replicationFactor == null, s"replicationFactor should be null, but it is $replicationFactor")
         require(replicasAssignments != null, s"replicaAssigments should not be null, but it is $replicasAssignments")
+
         replicasAssignments.asScala.foreach { case (partitionId, assignment) =>
           if (assignment.size < 2)
             throw new PolicyViolationException("Topic partitions should have at least 2 partitions, received " +
               s"${assignment.size} for partition $partitionId")
         }
       }
+
     }
 
-    def close(): Unit = ()
+    def close(): Unit = closed = true
+
   }
 }