You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by jg...@apache.org on 2017/01/27 05:27:48 UTC
kafka git commit: MINOR: Close create topics policy during shutdown
and more tests
Repository: kafka
Updated Branches:
refs/heads/trunk a01b72236 -> 2f9e9195a
MINOR: Close create topics policy during shutdown and more tests
Author: Ismael Juma <is...@juma.me.uk>
Reviewers: Jason Gustafson <ja...@confluent.io>
Closes #2443 from ijuma/close-create-topics-policy-during-shutdown
Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/2f9e9195
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/2f9e9195
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/2f9e9195
Branch: refs/heads/trunk
Commit: 2f9e9195abada7c2bcbd41147cb9d4c00f3a712d
Parents: a01b722
Author: Ismael Juma <is...@juma.me.uk>
Authored: Thu Jan 26 21:27:44 2017 -0800
Committer: Jason Gustafson <ja...@confluent.io>
Committed: Thu Jan 26 21:27:44 2017 -0800
----------------------------------------------------------------------
.../main/scala/kafka/server/AdminManager.scala | 1 +
.../CreateTopicsRequestWithPolicyTest.scala | 49 ++++++++++++++++----
2 files changed, 42 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kafka/blob/2f9e9195/core/src/main/scala/kafka/server/AdminManager.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/kafka/server/AdminManager.scala b/core/src/main/scala/kafka/server/AdminManager.scala
index 6dc224d..98a4b18 100644
--- a/core/src/main/scala/kafka/server/AdminManager.scala
+++ b/core/src/main/scala/kafka/server/AdminManager.scala
@@ -191,5 +191,6 @@ class AdminManager(val config: KafkaConfig,
def shutdown() {
topicPurgatory.shutdown()
+ CoreUtils.swallow(createTopicPolicy.foreach(_.close()))
}
}
http://git-wip-us.apache.org/repos/asf/kafka/blob/2f9e9195/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
index 9affea4..f2020cf 100644
--- a/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
+++ b/core/src/test/scala/unit/kafka/server/CreateTopicsRequestWithPolicyTest.scala
@@ -20,6 +20,7 @@ package kafka.server
import java.util
import java.util.Properties
+import kafka.log.LogConfig
import kafka.utils.TestUtils
import org.apache.kafka.common.errors.PolicyViolationException
import org.apache.kafka.common.protocol.Errors
@@ -48,9 +49,13 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
validateValidCreateTopicsRequests(new CreateTopicsRequest.Builder(
Map("topic2" -> new CreateTopicsRequest.TopicDetails(5, 3.toShort)).asJava, timeout, true).build())
+ val configs = Map(LogConfig.RetentionMsProp -> 4999.toString)
+ validateValidCreateTopicsRequests(new CreateTopicsRequest.Builder(
+ Map("topic3" -> new CreateTopicsRequest.TopicDetails(11, 2.toShort, configs.asJava)).asJava, timeout, true).build())
+
val assignments = replicaAssignmentToJava(Map(0 -> List(1, 0), 1 -> List(0, 1)))
validateValidCreateTopicsRequests(new CreateTopicsRequest.Builder(
- Map("topic3" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build())
+ Map("topic4" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build())
}
@Test
@@ -65,14 +70,23 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
Map("policy-topic1" -> error(Errors.POLICY_VIOLATION, Some("Topics should have at least 5 partitions, received 4"))))
validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
- Map("policy-topic2" -> new CreateTopicsRequest.TopicDetails(4, 1.toShort)).asJava, timeout, true).build(),
+ Map("policy-topic2" -> new CreateTopicsRequest.TopicDetails(4, 3.toShort)).asJava, timeout, true).build(),
Map("policy-topic2" -> error(Errors.POLICY_VIOLATION, Some("Topics should have at least 5 partitions, received 4"))))
+ val configs = Map(LogConfig.RetentionMsProp -> 5001.toString)
+ validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
+ Map("policy-topic3" -> new CreateTopicsRequest.TopicDetails(11, 2.toShort, configs.asJava)).asJava, timeout, true).build(),
+ Map("policy-topic3" -> error(Errors.POLICY_VIOLATION, Some("RetentionMs should be less than 5000ms if replicationFactor > 5"))))
+
+ validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
+ Map("policy-topic4" -> new CreateTopicsRequest.TopicDetails(11, 3.toShort, Map.empty.asJava)).asJava, timeout, true).build(),
+ Map("policy-topic4" -> error(Errors.POLICY_VIOLATION, Some("RetentionMs should be less than 5000ms if replicationFactor > 5"))))
+
val assignments = replicaAssignmentToJava(Map(0 -> List(1), 1 -> List(0)))
validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
- Map("policy-topic3" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build(),
- Map("policy-topic3" -> error(Errors.POLICY_VIOLATION,
- Some("""Topic partitions should have at least 2 partitions, received 1 for partition 0"""))), checkErrorMessage = true)
+ Map("policy-topic5" -> new CreateTopicsRequest.TopicDetails(assignments)).asJava, timeout).build(),
+ Map("policy-topic5" -> error(Errors.POLICY_VIOLATION,
+ Some("Topic partitions should have at least 2 partitions, received 1 for partition 0"))))
// Check that basic errors still work
validateErrorCreateTopicsRequests(new CreateTopicsRequest.Builder(
@@ -94,29 +108,48 @@ class CreateTopicsRequestWithPolicyTest extends AbstractCreateTopicsRequestTest
object CreateTopicsRequestWithPolicyTest {
class Policy extends CreateTopicPolicy {
- def configure(configs: util.Map[String, _]): Unit = ()
+
+ var configs: Map[String, _] = _
+ var closed = false
+
+ def configure(configs: util.Map[String, _]): Unit = {
+ this.configs = configs.asScala.toMap
+ }
def validate(requestMetadata: RequestMetadata): Unit = {
+ require(!closed, "Policy should not be closed")
+ require(!configs.isEmpty, "configure should have been called with non empty configs")
+
import requestMetadata._
- require(configs.isEmpty, s"Topic configs should be empty, but it is $configs")
if (numPartitions != null || replicationFactor != null) {
require(numPartitions != null, s"numPartitions should not be null, but it is $numPartitions")
require(replicationFactor != null, s"replicationFactor should not be null, but it is $replicationFactor")
require(replicasAssignments == null, s"replicaAssigments should be null, but it is $replicasAssignments")
+
if (numPartitions < 5)
throw new PolicyViolationException(s"Topics should have at least 5 partitions, received $numPartitions")
+
+ if (numPartitions > 10) {
+ if (requestMetadata.configs.asScala.get(LogConfig.RetentionMsProp).fold(true)(_.toInt > 5000))
+ throw new PolicyViolationException("RetentionMs should be less than 5000ms if replicationFactor > 5")
+ } else
+ require(requestMetadata.configs.isEmpty, s"Topic configs should be empty, but it is ${requestMetadata.configs}")
+
} else {
require(numPartitions == null, s"numPartitions should be null, but it is $numPartitions")
require(replicationFactor == null, s"replicationFactor should be null, but it is $replicationFactor")
require(replicasAssignments != null, s"replicaAssigments should not be null, but it is $replicasAssignments")
+
replicasAssignments.asScala.foreach { case (partitionId, assignment) =>
if (assignment.size < 2)
throw new PolicyViolationException("Topic partitions should have at least 2 partitions, received " +
s"${assignment.size} for partition $partitionId")
}
}
+
}
- def close(): Unit = ()
+ def close(): Unit = closed = true
+
}
}