You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by na...@apache.org on 2015/06/17 01:12:29 UTC

samza git commit: SAMZA-546 Add config option max.in.flight.requests.per.connection

Repository: samza
Updated Branches:
  refs/heads/master 0fc6ad255 -> e7cd142ea


SAMZA-546 Add config option max.in.flight.requests.per.connection


Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/e7cd142e
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/e7cd142e
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/e7cd142e

Branch: refs/heads/master
Commit: e7cd142eaf503ddb325ecbc24c3f27c0b39c13d5
Parents: 0fc6ad2
Author: rzuljevic <r....@levi9.com>
Authored: Tue Jun 16 16:11:33 2015 -0700
Committer: Navina <na...@gmail.com>
Committed: Tue Jun 16 16:11:33 2015 -0700

----------------------------------------------------------------------
 .../org/apache/samza/config/KafkaConfig.scala   |  24 ++--
 .../apache/samza/config/TestKafkaConfig.scala   | 124 ++++++++++++++-----
 .../test/integration/TestStatefulTask.scala     |   4 +-
 3 files changed, 113 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/samza/blob/e7cd142e/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
----------------------------------------------------------------------
diff --git a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
index a1de887..798033c 100644
--- a/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
+++ b/samza-kafka/src/main/scala/org/apache/samza/config/KafkaConfig.scala
@@ -179,13 +179,23 @@ class KafkaProducerConfig(val systemName: String,
       producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, byteArraySerializerClassName)
     }
 
-    // Always set (new) producer config for max_in_flight_requests_per_connection and retries_config to 1 & INT.MaxValue
-    // so that the producer does not optimistically send batches asychronously and thereby, messing up the ordering of outgoing messages
-    // Retries config is set to Max so that when all attempts fail, Samza also fails the send. We do not have any special handler
-    // for producer failure
-    // DO NOT let caller to override these 2 configs for Kafka
-    producerProperties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT)
-    producerProperties.put(ProducerConfig.RETRIES_CONFIG, RETRIES_DEFAULT)
+    if(producerProperties.containsKey(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION)
+        && producerProperties.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION).asInstanceOf[String].toInt > MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT) {
+      warn("Setting '%s' to a value other than %d does not guarantee message ordering because new messages will be sent without waiting for previous ones to be acknowledged." 
+          format (ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT))
+    } else {
+      producerProperties.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT)
+    }
+
+    if(producerProperties.containsKey(ProducerConfig.RETRIES_CONFIG) 
+        && producerProperties.get(ProducerConfig.RETRIES_CONFIG).asInstanceOf[String].toInt < RETRIES_DEFAULT) {
+        warn("Samza does not provide producer failure handling. Consider setting '%s' to a large value, like Int.MAX." format ProducerConfig.RETRIES_CONFIG)
+    } else {
+      // Retries config is set to Max so that when all attempts fail, Samza also fails the send. We do not have any special handler
+      // for producer failure
+      producerProperties.put(ProducerConfig.RETRIES_CONFIG, RETRIES_DEFAULT)
+    }
+    
     producerProperties
   }
 

http://git-wip-us.apache.org/repos/asf/samza/blob/e7cd142e/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
----------------------------------------------------------------------
diff --git a/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala b/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
index 95f76b6..85badf9 100644
--- a/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
+++ b/samza-kafka/src/test/scala/org/apache/samza/config/TestKafkaConfig.scala
@@ -22,32 +22,41 @@ package org.apache.samza.config
 import java.net.URI
 import java.io.File
 import java.util.Properties
-
 import kafka.consumer.ConsumerConfig
-
 import org.apache.samza.config.factories.PropertiesConfigFactory
 import org.junit.Assert._
 import org.junit.Test
-
 import scala.collection.JavaConversions._
+import org.apache.kafka.common.serialization.ByteArraySerializer
+import org.apache.kafka.clients.producer.ProducerConfig
+import org.junit.Before
+import org.junit.BeforeClass
 
 class TestKafkaConfig {
-
+  
+  var props : Properties = new Properties
+  val SYSTEM_NAME = "kafka";
+  val KAFKA_PRODUCER_PROPERTY_PREFIX = "systems." + SYSTEM_NAME + ".producer."
+  
+  @Before
+  def setupProperties() {
+    props = new Properties
+    props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + "bootstrap.servers", "localhost:9092")
+    props.setProperty("systems." + SYSTEM_NAME + ".consumer.zookeeper.connect", "localhost:2181/")
+  }
+  
   @Test
   def testIdGeneration = {
     val factory = new PropertiesConfigFactory()
-    val props = new Properties
-    props.setProperty(" systems.kafka.samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory")
-    props.setProperty("systems.kafka.consumer.zookeeper.connect", "localhost:2181/")
-    props.setProperty("systems.kafka.producer.bootstrap.servers", "localhost:9092")
+    props.setProperty("systems." + SYSTEM_NAME + ".samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory")
 
     val mapConfig = new MapConfig(props.toMap[String, String])
     val kafkaConfig = new KafkaConfig(mapConfig)
 
-    val consumerConfig1 = kafkaConfig.getKafkaSystemConsumerConfig("kafka")
+    val consumerConfig1 = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME)
     val consumerClientId1 = consumerConfig1.clientId
     val groupId1 = consumerConfig1.groupId
-    val consumerConfig2 = kafkaConfig.getKafkaSystemConsumerConfig("kafka")
+    val consumerConfig2 = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME)
     val consumerClientId2 = consumerConfig2.clientId
     val groupId2 = consumerConfig2.groupId
     assert(consumerClientId1.startsWith("undefined-samza-consumer-"))
@@ -57,22 +66,22 @@ class TestKafkaConfig {
     assert(consumerClientId1 != consumerClientId2)
     assert(groupId1 != groupId2)
 
-    val consumerConfig3 = kafkaConfig.getKafkaSystemConsumerConfig("kafka", "TestClientId", "TestGroupId")
+    val consumerConfig3 = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME, "TestClientId", "TestGroupId")
     val consumerClientId3 = consumerConfig3.clientId
     val groupId3 = consumerConfig3.groupId
     assert(consumerClientId3 == "TestClientId")
     assert(groupId3 == "TestGroupId")
 
-    val producerConfig1 = kafkaConfig.getKafkaSystemProducerConfig("kafka")
+    val producerConfig1 = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
     val producerClientId1 = producerConfig1.clientId
-    val producerConfig2 = kafkaConfig.getKafkaSystemProducerConfig("kafka")
+    val producerConfig2 = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
     val producerClientId2 = producerConfig2.clientId
 
     assert(producerClientId1.startsWith("undefined-samza-producer-"))
     assert(producerClientId2.startsWith("undefined-samza-producer-"))
     assert(producerClientId1 != producerClientId2)
 
-    val producerConfig3 = kafkaConfig.getKafkaSystemProducerConfig("kafka", "TestClientId")
+    val producerConfig3 = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME, "TestClientId")
     val producerClientId3 = producerConfig3.clientId
     assert(producerClientId3 == "TestClientId")
 
@@ -80,40 +89,34 @@ class TestKafkaConfig {
 
   @Test
   def testStreamLevelFetchSizeOverride() {
-    val props = new Properties
-    props.setProperty("systems.kafka.consumer.zookeeper.connect", "localhost:2181/")
-    props.setProperty("systems.kafka.producer.bootstrap.servers", "localhost:9092")
-
     val mapConfig = new MapConfig(props.toMap[String, String])
     val kafkaConfig = new KafkaConfig(mapConfig)
-    val consumerConfig = kafkaConfig.getKafkaSystemConsumerConfig("kafka")
+    val consumerConfig = kafkaConfig.getKafkaSystemConsumerConfig(SYSTEM_NAME)
     // default fetch size
     assertEquals(1024*1024, consumerConfig.fetchMessageMaxBytes)
 
-    props.setProperty("systems.kafka.consumer.fetch.message.max.bytes", "262144")
+    props.setProperty("systems." + SYSTEM_NAME + ".consumer.fetch.message.max.bytes", "262144")
     val mapConfig1 = new MapConfig(props.toMap[String, String])
     val kafkaConfig1 = new KafkaConfig(mapConfig1)
-    val consumerConfig1 = kafkaConfig1.getKafkaSystemConsumerConfig("kafka")
+    val consumerConfig1 = kafkaConfig1.getKafkaSystemConsumerConfig(SYSTEM_NAME)
     // shared fetch size
     assertEquals(512*512, consumerConfig1.fetchMessageMaxBytes)
     
-    props.setProperty("systems.kafka.streams.topic1.consumer.fetch.message.max.bytes", "65536")
+    props.setProperty("systems." + SYSTEM_NAME + ".streams.topic1.consumer.fetch.message.max.bytes", "65536")
     val mapConfig2 = new MapConfig(props.toMap[String, String])
     val kafkaConfig2 = new KafkaConfig(mapConfig2)
-    val consumerConfig2 = kafkaConfig2.getFetchMessageMaxBytesTopics("kafka")
+    val consumerConfig2 = kafkaConfig2.getFetchMessageMaxBytesTopics(SYSTEM_NAME)
     // topic fetch size
     assertEquals(256*256, consumerConfig2 getOrElse ("topic1", 1024*1024))
   }
 
   @Test
   def testChangeLogProperties() {
-    val props = (new Properties /: Map(
-      "systems.kafka.samza.factory" -> "org.apache.samza.system.kafka.KafkaSystemFactory",
-      "stores.test1.changelog" -> "kafka.mychangelog1",
-      "stores.test2.changelog" -> "kafka.mychangelog2",
-      "stores.test1.changelog.kafka.cleanup.policy" -> "delete"
-      )) { case (props, (k, v)) => props.put(k, v); props }
-
+    props.setProperty("systems." + SYSTEM_NAME + ".samza.factory", "org.apache.samza.system.kafka.KafkaSystemFactory")
+    props.setProperty("stores.test1.changelog", "kafka.mychangelog1")
+    props.setProperty("stores.test2.changelog", "kafka.mychangelog2")
+    props.setProperty("stores.test1.changelog.kafka.cleanup.policy", "delete")
+    
     val mapConfig = new MapConfig(props.toMap[String, String])
     val kafkaConfig = new KafkaConfig(mapConfig)
     assertEquals(kafkaConfig.getChangelogKafkaProperties("test1").getProperty("cleanup.policy"), "delete")
@@ -122,4 +125,65 @@ class TestKafkaConfig {
     assertEquals(storeToChangelog.get("test1").getOrElse(""), "mychangelog1")
     assertEquals(storeToChangelog.get("test2").getOrElse(""), "mychangelog2")
   }
+  
+  @Test
+  def testDefaultValuesForProducerProperties() {
+    val mapConfig = new MapConfig(props.toMap[String, String])
+    val kafkaConfig = new KafkaConfig(mapConfig)
+    val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+    val producerProperties = kafkaProducerConfig.getProducerProperties
+    
+    assertEquals(classOf[ByteArraySerializer].getCanonicalName, producerProperties.get(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG))
+    assertEquals(classOf[ByteArraySerializer].getCanonicalName, producerProperties.get(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG))
+    assertEquals(kafkaProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION_DEFAULT, producerProperties.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION))
+    assertEquals(kafkaProducerConfig.RETRIES_DEFAULT, producerProperties.get(ProducerConfig.RETRIES_CONFIG))
+  }
+
+  @Test
+  def testMaxInFlightRequestsPerConnectionOverride() {
+    val expectedValue = "200";
+    
+    props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, expectedValue);
+    
+    val mapConfig = new MapConfig(props.toMap[String, String])
+    val kafkaConfig = new KafkaConfig(mapConfig)
+    val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+    val producerProperties = kafkaProducerConfig.getProducerProperties
+    
+    assertEquals(expectedValue, producerProperties.get(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION))
+  }
+  
+  @Test
+  def testRetriesOverride() {
+    val expectedValue = "200";
+    
+    props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.RETRIES_CONFIG, expectedValue);
+    
+    val mapConfig = new MapConfig(props.toMap[String, String])
+    val kafkaConfig = new KafkaConfig(mapConfig)
+    val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+    val producerProperties = kafkaProducerConfig.getProducerProperties
+    
+    assertEquals(expectedValue, producerProperties.get(ProducerConfig.RETRIES_CONFIG))
+  }
+  
+  @Test(expected = classOf[NumberFormatException])
+  def testMaxInFlightRequestsPerConnectionWrongNumberFormat() {
+    props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "Samza");
+    
+    val mapConfig = new MapConfig(props.toMap[String, String])
+    val kafkaConfig = new KafkaConfig(mapConfig)
+    val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+    kafkaProducerConfig.getProducerProperties
+  }
+  
+  @Test(expected = classOf[NumberFormatException])
+  def testRetriesWrongNumberFormat() {
+    props.setProperty(KAFKA_PRODUCER_PROPERTY_PREFIX + ProducerConfig.RETRIES_CONFIG, "Samza");
+    
+    val mapConfig = new MapConfig(props.toMap[String, String])
+    val kafkaConfig = new KafkaConfig(mapConfig)
+    val kafkaProducerConfig = kafkaConfig.getKafkaSystemProducerConfig(SYSTEM_NAME)
+    kafkaProducerConfig.getProducerProperties
+  }
 }

http://git-wip-us.apache.org/repos/asf/samza/blob/e7cd142e/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
index 8200696..ea702a9 100644
--- a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
@@ -97,8 +97,8 @@ object TestStatefulTask {
   config.put("bootstrap.servers", brokers)
   config.put("request.required.acks", "-1")
   config.put("serializer.class", "kafka.serializer.StringEncoder")
-  config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, new Integer(1))
-  config.put(ProducerConfig.RETRIES_CONFIG, new Integer(Integer.MAX_VALUE-1))
+  config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
+  config.put(ProducerConfig.RETRIES_CONFIG, (new Integer(Integer.MAX_VALUE-1)).toString())
   val producerConfig = new KafkaProducerConfig("kafka", "i001", config)
   var producer: Producer[Array[Byte], Array[Byte]] = null
   val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123"))