You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by na...@apache.org on 2015/06/17 01:12:29 UTC
samza git commit: SAMZA-546 Add config option
max.in.flight.requests.per.connection
Repository: samza
Updated Branches:
refs/heads/master 0fc6ad255 -> e7cd142ea
SAMZA-546 Add config option max.in.flight.requests.per.connection
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/e7cd142e
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/e7cd142e
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/e7cd142e
Branch: refs/heads/master
Commit: e7cd142eaf503ddb325ecbc24c3f27c0b39c13d5
Parents: 0fc6ad2
Author: rzuljevic <r....@levi9.com>
Authored: Tue Jun 16 16:11:33 2015 -0700
Committer: Navina <na...@gmail.com>
Committed: Tue Jun 16 16:11:33 2015 -0700
----------------------------------------------------------------------
.../org/apache/samza/config/KafkaConfig.scala | 24 ++--
.../apache/samza/config/TestKafkaConfig.scala | 124 ++++++++++++++-----
.../test/integration/TestStatefulTask.scala | 4 +-
3 files changed, 113 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/e7cd142e/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
----------------------------------------------------------------------
diff --git a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
index a1de887..798033c 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
@@ -179,13 +179,23 @@ class KafkaProducerConfig(val systemName: String,
producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, byteArraySerializerClassName)
}
- // Always set (new) producer config for max_in_flight_requests_per_connection and retries_config to 1 & INT.MaxValue
- // so that the producer does not optimistically send batches asychronously and thereby, messing up the ordering of outgoing messages
- // Retries config is set to Max so that when all attempts fail, Samza also fails the send. We do not have any special handler
- // for producer failure
- // DO NOT let caller to override these 2 configs for Kafka
- producerProperties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT)
- producerProperties.put(ProducerConfig.RETRIES_CONFIG, RETRIES_DEFAULT)
+ if(producerProperties.containsKey(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)
+ && producerProperties.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION).asInstanceOf[String].toInt > MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT) {
+ warn("Setting '%s' to a value other than %d does not guarantee message ordering because new messages will be sent without waiting for previous ones to be acknowledged."
+ format (ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT))
+ } else {
+ producerProperties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT)
+ }
+
+ if(producerProperties.containsKey(ProducerConfig.RETRIES_CONFIG)
+ && producerProperties.get(ProducerConfig.RETRIES_CONFIG).asInstanceOf[String].toInt < RETRIES_DEFAULT) {
+ warn("Samza does not provide producer failure handling. Consider setting '%s' to a large value, like Int.MAX." format ProducerConfig.RETRIES_CONFIG)
+ } else {
+ // Retries config is set to Max so that when all attempts fail, Samza also fails the send. We do not have any special handler
+ // for producer failure
+ producerProperties.put(ProducerConfig.RETRIES_CONFIG, RETRIES_DEFAULT)
+ }
+
producerProperties
}
http://git-wip-us.apache.org/repos/asf/samza/blob/e7cd142e/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
----------------------------------------------------------------------
diff --git a/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala b/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
index 95f76b6..85badf9 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
@@ -22,32 +22,41 @@ package org.apache.samza.config
import java.net.URI
import java.io.File
import java.util.Properties
-
import kafka.consumer.ConsumerConfig
-
import org.apache.samza.config.factories.PropertiesConfigFactory
import org.junit.Assert._
import org.junit.Test
-
import scala.collection.JavaConversions._
+import org.apache.kafka.common.serialization.ByteArraySerializer
+import org.apache.kafka.clients.producer.ProducerConfig
+import org.junit.Before
+import org.junit.BeforeClass
class TestKafkaConfig {
-
+
+ var props : Properties = new Properties
+ val SYSTEM_NAME = "kafka";
+ val KAFKA_PRODUCER_PROPERTY_PREFIX = "systems." + SYSTEM_NAME + ".producer."
+
+ @Before
+ def setupProperties() {
+ props = new Properties
+ props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + "bootstrap.servers", "localhost:9092")
+ props.setProperty("systems." + SYSTEM_NAME + ".consumer.zookeeper.connect", "localhost:2181/")
+ }
+
@Test
def testIdGeneration = {
val factory = new PropertiesConfigFactory()
- val props = new Properties
- props.setProperty(" systems.kafka.samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory")
- props.setProperty("systems.kafka.consumer.zookeeper.connect", "localhost:2181/")
- props.setProperty("systems.kafka.producer.bootstrap.servers", "localhost:9092")
+ props.setProperty("systems." + SYSTEM_NAME + ".samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory")
val mapConfig = new MapConfig(props.toMap[String, String])
val kafkaConfig = new KafkaConfig(mapConfig)
- val consumerConfig1 = kafkaConfig.getKafkaSystemConsumerConfig("kafka")
+ val consumerConfig1 = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME)
val consumerClientId1 = consumerConfig1.clientId
val groupId1 = consumerConfig1.groupId
- val consumerConfig2 = kafkaConfig.getKafkaSystemConsumerConfig("kafka")
+ val consumerConfig2 = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME)
val consumerClientId2 = consumerConfig2.clientId
val groupId2 = consumerConfig2.groupId
assert(consumerClientId1.startsWith("undefined-samza-consumer-"))
@@ -57,22 +66,22 @@ class TestKafkaConfig {
assert(consumerClientId1 != consumerClientId2)
assert(groupId1 != groupId2)
- val consumerConfig3 = kafkaConfig.getKafkaSystemConsumerConfig("kafka", "TestClientId", "TestGroupId")
+ val consumerConfig3 = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME, "TestClientId", "TestGroupId")
val consumerClientId3 = consumerConfig3.clientId
val groupId3 = consumerConfig3.groupId
assert(consumerClientId3 == "TestClientId")
assert(groupId3 == "TestGroupId")
- val producerConfig1 = kafkaConfig.getKafkaSystemProducerConfig("kafka")
+ val producerConfig1 = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
val producerClientId1 = producerConfig1.clientId
- val producerConfig2 = kafkaConfig.getKafkaSystemProducerConfig("kafka")
+ val producerConfig2 = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
val producerClientId2 = producerConfig2.clientId
assert(producerClientId1.startsWith("undefined-samza-producer-"))
assert(producerClientId2.startsWith("undefined-samza-producer-"))
assert(producerClientId1 != producerClientId2)
- val producerConfig3 = kafkaConfig.getKafkaSystemProducerConfig("kafka", "TestClientId")
+ val producerConfig3 = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME, "TestClientId")
val producerClientId3 = producerConfig3.clientId
assert(producerClientId3 == "TestClientId")
@@ -80,40 +89,34 @@ class TestKafkaConfig {
@Test
def testStreamLevelFetchSizeOverride() {
- val props = new Properties
- props.setProperty("systems.kafka.consumer.zookeeper.connect", "localhost:2181/")
- props.setProperty("systems.kafka.producer.bootstrap.servers", "localhost:9092")
-
val mapConfig = new MapConfig(props.toMap[String, String])
val kafkaConfig = new KafkaConfig(mapConfig)
- val consumerConfig = kafkaConfig.getKafkaSystemConsumerConfig("kafka")
+ val consumerConfig = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME)
// default fetch size
assertEquals(1024*1024, consumerConfig.fetchMessageMaxBytes)
- props.setProperty("systems.kafka.consumer.fetch.message.max.bytes", "262144")
+ props.setProperty("systems." + SYSTEM_NAME + ".consumer.fetch.message.max.bytes", "262144")
val mapConfig1 = new MapConfig(props.toMap[String, String])
val kafkaConfig1 = new KafkaConfig(mapConfig1)
- val consumerConfig1 = kafkaConfig1.getKafkaSystemConsumerConfig("kafka")
+ val consumerConfig1 = kafkaConfig1.getKafkaSystemConsumerConfig(SYSTEM_NAME)
// shared fetch size
assertEquals(512*512, consumerConfig1.fetchMessageMaxBytes)
- props.setProperty("systems.kafka.streams.topic1.consumer.fetch.message.max.bytes", "65536")
+ props.setProperty("systems." + SYSTEM_NAME + ".streams.topic1.consumer.fetch.message.max.bytes", "65536")
val mapConfig2 = new MapConfig(props.toMap[String, String])
val kafkaConfig2 = new KafkaConfig(mapConfig2)
- val consumerConfig2 = kafkaConfig2.getFetchMessageMaxBytesTopics("kafka")
+ val consumerConfig2 = kafkaConfig2.getFetchMessageMaxBytesTopics(SYSTEM_NAME)
// topic fetch size
assertEquals(256*256, consumerConfig2 getOrElse ("topic1", 1024*1024))
}
@Test
def testChangeLogProperties() {
- val props = (new Properties /: Map(
- "systems.kafka.samza.factory" -> "org.apache.samza.system.kafka.KafkaSystemFactory",
- "stores.test1.changelog" -> "kafka.mychangelog1",
- "stores.test2.changelog" -> "kafka.mychangelog2",
- "stores.test1.changelog.kafka.cleanup.policy" -> "delete"
- )) { case (props, (k, v)) => props.put(k, v); props }
-
+ props.setProperty("systems." + SYSTEM_NAME + ".samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory")
+ props.setProperty("stores.test1.changelog", "kafka.mychangelog1")
+ props.setProperty("stores.test2.changelog", "kafka.mychangelog2")
+ props.setProperty("stores.test1.changelog.kafka.cleanup.policy", "delete")
+
val mapConfig = new MapConfig(props.toMap[String, String])
val kafkaConfig = new KafkaConfig(mapConfig)
assertEquals(kafkaConfig.getChangelogKafkaProperties("test1").getProperty("cleanup.policy"), "delete")
@@ -122,4 +125,65 @@ class TestKafkaConfig {
assertEquals(storeToChangelog.get("test1").getOrElse(""), "mychangelog1")
assertEquals(storeToChangelog.get("test2").getOrElse(""), "mychangelog2")
}
+
+ @Test
+ def testDefaultValuesForProducerProperties() {
+ val mapConfig = new MapConfig(props.toMap[String, String])
+ val kafkaConfig = new KafkaConfig(mapConfig)
+ val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+ val producerProperties = kafkaProducerConfig.getProducerProperties
+
+ assertEquals(classOf[ByteArraySerializer].getCanonicalName, producerProperties.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG))
+ assertEquals(classOf[ByteArraySerializer].getCanonicalName, producerProperties.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG))
+ assertEquals(kafkaProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT, producerProperties.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION))
+ assertEquals(kafkaProducerConfig.RETRIES_DEFAULT, producerProperties.get(ProducerConfig.RETRIES_CONFIG))
+ }
+
+ @Test
+ def testMaxInFlightRequestsPerConnectionOverride() {
+ val expectedValue = "200";
+
+ props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, expectedValue);
+
+ val mapConfig = new MapConfig(props.toMap[String, String])
+ val kafkaConfig = new KafkaConfig(mapConfig)
+ val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+ val producerProperties = kafkaProducerConfig.getProducerProperties
+
+ assertEquals(expectedValue, producerProperties.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION))
+ }
+
+ @Test
+ def testRetriesOverride() {
+ val expectedValue = "200";
+
+ props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.RETRIES_CONFIG, expectedValue);
+
+ val mapConfig = new MapConfig(props.toMap[String, String])
+ val kafkaConfig = new KafkaConfig(mapConfig)
+ val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+ val producerProperties = kafkaProducerConfig.getProducerProperties
+
+ assertEquals(expectedValue, producerProperties.get(ProducerConfig.RETRIES_CONFIG))
+ }
+
+ @Test(expected = classOf[NumberFormatException])
+ def testMaxInFlightRequestsPerConnectionWrongNumberFormat() {
+ props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "Samza");
+
+ val mapConfig = new MapConfig(props.toMap[String, String])
+ val kafkaConfig = new KafkaConfig(mapConfig)
+ val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+ kafkaProducerConfig.getProducerProperties
+ }
+
+ @Test(expected = classOf[NumberFormatException])
+ def testRetriesWrongNumberFormat() {
+ props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.RETRIES_CONFIG, "Samza");
+
+ val mapConfig = new MapConfig(props.toMap[String, String])
+ val kafkaConfig = new KafkaConfig(mapConfig)
+ val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+ kafkaProducerConfig.getProducerProperties
+ }
}
http://git-wip-us.apache.org/repos/asf/samza/blob/e7cd142e/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
index 8200696..ea702a9 100644
--- a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
@@ -97,8 +97,8 @@ object TestStatefulTask {
config.put("bootstrap.servers", brokers)
config.put("request.required.acks", "-1")
config.put("serializer.class", "kafka.serializer.StringEncoder")
- config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, new Integer(1))
- config.put(ProducerConfig.RETRIES_CONFIG, new Integer(Integer.MAX_VALUE-1))
+ config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
+ config.put(ProducerConfig.RETRIES_CONFIG, (new Integer(Integer.MAX_VALUE-1)).toString())
val producerConfig = new KafkaProducerConfig("kafka", "i001", config)
var producer: Producer[Array[Byte], Array[Byte]] = null
val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123"))