You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by ni...@apache.org on 2015/08/08 02:05:22 UTC

samza git commit: SAMZA-744: Fix the container shutdown sequence s.t. KV-stores are flushed before shutting down producers

Repository: samza
Updated Branches:
  refs/heads/master c32022498 -> 98df876dd


SAMZA-744: Fix the container shutdown sequence s.t. KV-stores are flushed before shutting down producers


Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/98df876d
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/98df876d
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/98df876d

Branch: refs/heads/master
Commit: 98df876dd4492b7d32422de0eccff1cbc26e1888
Parents: c320224
Author: Yi Pan <ni...@gmail.com>
Authored: Fri Aug 7 17:05:08 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Fri Aug 7 17:05:08 2015 -0700

----------------------------------------------------------------------
 build.gradle                                    |   2 +-
 .../apache/samza/container/SamzaContainer.scala |   2 +-
 .../test/integration/StreamTaskTestUtil.scala   | 335 +++++++++++++++++++
 .../integration/TestShutdownStatefulTask.scala  | 139 ++++++++
 .../test/integration/TestStatefulTask.scala     | 333 ++----------------
 5 files changed, 511 insertions(+), 300 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index cc30b45..a935088 100644
--- a/build.gradle
+++ b/build.gradle
@@ -78,7 +78,7 @@ rat {
     'gradle/wrapper/**',
     'gradlew',
     'gradlew.bat',
-    'samza-test/state/mystore/**',
+    'samza-test/state/**',
     'README.md',
     'RELEASE.md',
     'samza-test/src/main/resources/**',

http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
index 702ce26..85b012b 100644
--- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
+++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
@@ -581,8 +581,8 @@ class SamzaContainer(
 
       shutdownConsumers
       shutdownTask
-      shutdownProducers
       shutdownStores
+      shutdownProducers
       shutdownLocalityManager
       shutdownOffsetManager
       shutdownMetrics

http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
new file mode 100644
index 0000000..c6e994f
--- /dev/null
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.test.integration
+
+import java.util
+import java.util.Properties
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import kafka.admin.AdminUtils
+import kafka.consumer.{Consumer, ConsumerConfig}
+import kafka.message.MessageAndMetadata
+import kafka.server.{KafkaConfig, KafkaServer}
+import kafka.utils.{TestUtils, TestZKUtils, Utils, ZKStringSerializer}
+import kafka.zk.EmbeddedZookeeper
+import org.I0Itec.zkclient.ZkClient
+import org.apache.kafka.clients.producer.{KafkaProducer, Producer, ProducerConfig, ProducerRecord}
+import org.apache.samza.Partition
+import org.apache.samza.checkpoint.Checkpoint
+import org.apache.samza.config.{Config, KafkaProducerConfig, MapConfig}
+import org.apache.samza.container.TaskName
+import org.apache.samza.job.local.ThreadJobFactory
+import org.apache.samza.job.{ApplicationStatus, JobRunner, StreamJob}
+import org.apache.samza.system.kafka.TopicMetadataCache
+import org.apache.samza.system.{IncomingMessageEnvelope, SystemStreamPartition}
+import org.apache.samza.task._
+import org.apache.samza.util.{ClientUtilTopicMetadataStore, KafkaUtil, TopicMetadataStore}
+import org.junit.Assert._
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.{ArrayBuffer, HashMap, SynchronizedMap}
+
+/*
+ * This creates an singleton instance of TestBaseStreamTask and implement the helper functions to
+ * 1. start the local ZooKeeper server
+ * 2. start the local Kafka brokers
+ * 3. create and validate test topics
+ * 4. shutdown servers and cleanup test directories and files
+ */
+object StreamTaskTestUtil {
+  val INPUT_TOPIC = "input"
+  val TOTAL_TASK_NAMES = 1
+  val REPLICATION_FACTOR = 3
+
+  val zkConnect: String = TestZKUtils.zookeeperConnect
+  var zkClient: ZkClient = null
+  val zkConnectionTimeout = 6000
+  val zkSessionTimeout = 6000
+
+  val brokerId1 = 0
+  val brokerId2 = 1
+  val brokerId3 = 2
+  val ports = TestUtils.choosePorts(3)
+  val (port1, port2, port3) = (ports(0), ports(1), ports(2))
+
+  val props1 = TestUtils.createBrokerConfig(brokerId1, port1)
+  val props2 = TestUtils.createBrokerConfig(brokerId2, port2)
+  val props3 = TestUtils.createBrokerConfig(brokerId3, port3)
+  props1.setProperty("auto.create.topics.enable","false")
+  props2.setProperty("auto.create.topics.enable","false")
+  props3.setProperty("auto.create.topics.enable","false")
+
+  val config = new util.HashMap[String, Object]()
+  val brokers = "localhost:%d,localhost:%d,localhost:%d" format (port1, port2, port3)
+  config.put("bootstrap.servers", brokers)
+  config.put("request.required.acks", "-1")
+  config.put("serializer.class", "kafka.serializer.StringEncoder")
+  config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
+  config.put(ProducerConfig.RETRIES_CONFIG, (new Integer(Integer.MAX_VALUE-1)).toString())
+  val producerConfig = new KafkaProducerConfig("kafka", "i001", config)
+  var producer: Producer[Array[Byte], Array[Byte]] = null
+  val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123"))
+  val cp2 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "12345"))
+  var zookeeper: EmbeddedZookeeper = null
+  var server1: KafkaServer = null
+  var server2: KafkaServer = null
+  var server3: KafkaServer = null
+  var metadataStore: TopicMetadataStore = null
+
+  /*
+   * This is the default job configuration. Each test class can override the default configuration below.
+   */
+  var jobConfig = Map(
+    "job.factory.class" -> classOf[ThreadJobFactory].getCanonicalName,
+    "task.inputs" -> "kafka.input",
+    "serializers.registry.string.class" -> "org.apache.samza.serializers.StringSerdeFactory",
+    "systems.kafka.samza.factory" -> "org.apache.samza.system.kafka.KafkaSystemFactory",
+    // Always start consuming at offset 0. This avoids a race condition between
+    // the producer and the consumer in this test (SAMZA-166, SAMZA-224).
+    "systems.kafka.samza.offset.default" -> "oldest", // applies to a nonempty topic
+    "systems.kafka.consumer.auto.offset.reset" -> "smallest", // applies to an empty topic
+    "systems.kafka.samza.msg.serde" -> "string",
+    "systems.kafka.consumer.zookeeper.connect" -> zkConnect,
+    "systems.kafka.producer.bootstrap.servers" -> ("localhost:%s" format port1),
+    // Since using state, need a checkpoint manager
+    // Due to SAMZA-754, the following section can not be removed yet.
+    "task.checkpoint.factory" -> "org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory",
+    "task.checkpoint.system" -> "kafka",
+    "task.checkpoint.replication.factor" -> "1",
+    // However, don't have the inputs use the checkpoint manager
+    // since the second part of the test expects to replay the input streams.
+    "systems.kafka.streams.input.samza.reset.offset" -> "false")
+
+  def apply(map: Map[String, String]): Unit = {
+    jobConfig ++= map
+    TestTask.reset()
+  }
+
+  def beforeSetupServers {
+    zookeeper = new EmbeddedZookeeper(zkConnect)
+    server1 = TestUtils.createServer(new KafkaConfig(props1))
+    server2 = TestUtils.createServer(new KafkaConfig(props2))
+    server3 = TestUtils.createServer(new KafkaConfig(props3))
+    zkClient = new ZkClient(zkConnect + "/", 6000, 6000, ZKStringSerializer)
+    producer = new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties)
+    metadataStore = new ClientUtilTopicMetadataStore(brokers, "some-job-name")
+
+    createTopics
+    validateTopics
+  }
+
+  def createTopics {
+    AdminUtils.createTopic(
+      zkClient,
+      INPUT_TOPIC,
+      TOTAL_TASK_NAMES,
+      REPLICATION_FACTOR)
+  }
+
+  def validateTopics {
+    val topics = Set(INPUT_TOPIC)
+    var done = false
+    var retries = 0
+
+    while (!done && retries < 100) {
+      try {
+        val topicMetadataMap = TopicMetadataCache.getTopicMetadata(topics, "kafka", metadataStore.getTopicInfo)
+
+        topics.foreach(topic => {
+          val topicMetadata = topicMetadataMap(topic)
+          val errorCode = topicMetadata.errorCode
+
+          KafkaUtil.maybeThrowException(errorCode)
+        })
+
+        done = true
+      } catch {
+        case e: Exception =>
+          System.err.println("Got exception while validating test topics. Waiting and retrying.", e)
+          retries += 1
+          Thread.sleep(500)
+      }
+    }
+
+    if (retries >= 100) {
+      fail("Unable to successfully create topics. Tried to validate %s times." format retries)
+    }
+  }
+
+  def afterCleanLogDirs {
+    producer.close()
+    server1.shutdown
+    server1.awaitShutdown()
+    server2.shutdown
+    server2.awaitShutdown()
+    server3.shutdown
+    server3.awaitShutdown()
+    Utils.rm(server1.config.logDirs)
+    Utils.rm(server2.config.logDirs)
+    Utils.rm(server3.config.logDirs)
+    zkClient.close
+    zookeeper.shutdown
+  }
+}
+
+/* This class implement the base utility to implement an integration test for StreamTask
+ * It implements helper functions to start/stop the job, send messages to a task, and read all messages from a topic
+ */
+class StreamTaskTestUtil {
+  import StreamTaskTestUtil._
+
+  /**
+   * Start a job for TestTask, and do some basic sanity checks around startup
+   * time, number of partitions, etc.
+   */
+  def startJob = {
+    // Start task.
+    val job = new JobRunner(new MapConfig(jobConfig)).run
+    assertEquals(ApplicationStatus.Running, job.waitForStatus(ApplicationStatus.Running, 60000))
+    TestTask.awaitTaskRegistered
+    val tasks = TestTask.tasks
+    assertEquals("Should only have a single partition in this task", 1, tasks.size)
+    val task = tasks.values.toList.head
+    task.initFinished.await(60, TimeUnit.SECONDS)
+    assertEquals(0, task.initFinished.getCount)
+    (job, task)
+  }
+
+  /**
+   * Kill a job, and wait for an unsuccessful finish (since this throws an
+   * interrupt, which is forwarded on to ThreadJob, and marked as a failure).
+   */
+  def stopJob(job: StreamJob) {
+    // Shutdown task.
+    job.kill
+    assertEquals(ApplicationStatus.UnsuccessfulFinish, job.waitForFinish(60000))
+  }
+
+  /**
+   * Send a message to the input topic, and validate that it gets to the test task.
+   */
+  def send(task: TestTask, msg: String) {
+    producer.send(new ProducerRecord(INPUT_TOPIC, msg.getBytes)).get()
+    task.awaitMessage
+    assertEquals(msg, task.received.last)
+  }
+
+  /**
+   * Read all messages from a topic starting from last saved offset for group.
+   * To read all from offset 0, specify a unique, new group string.
+   */
+  def readAll(topic: String, maxOffsetInclusive: Int, group: String): List[String] = {
+    val props = new Properties
+
+    props.put("zookeeper.connect", zkConnect)
+    props.put("group.id", group)
+    props.put("auto.offset.reset", "smallest")
+
+    val consumerConfig = new ConsumerConfig(props)
+    val consumerConnector = Consumer.create(consumerConfig)
+    var stream = consumerConnector.createMessageStreams(Map(topic -> 1)).get(topic).get.get(0).iterator
+    var message: MessageAndMetadata[Array[Byte], Array[Byte]] = null
+    var messages = ArrayBuffer[String]()
+
+    while (message == null || message.offset < maxOffsetInclusive) {
+      message = stream.next
+      if (message.message == null) {
+        messages += null
+      } else {
+        messages += new String(message.message, "UTF-8")
+      }
+      System.err.println("StreamTaskTestUtil.readAll(): offset=%s, message=%s" format (message.offset, messages.last))
+    }
+
+    consumerConnector.shutdown
+
+    messages.toList
+  }
+
+}
+
+object TestTask {
+  val tasks = new HashMap[TaskName, TestTask] with SynchronizedMap[TaskName, TestTask]
+  var totalTasks = 1
+  @volatile var allTasksRegistered = new CountDownLatch(totalTasks)
+
+  def reset(): Unit = {
+    TestTask.totalTasks = StreamTaskTestUtil.TOTAL_TASK_NAMES
+    TestTask.allTasksRegistered = new CountDownLatch(TestTask.totalTasks)
+  }
+
+  /**
+   * Static method that tasks can use to register themselves with. Useful so
+   * we don't have to sneak into the ThreadJob/SamzaContainer to get our test
+   * tasks.
+   */
+  def register(taskName: TaskName, task: TestTask) {
+    tasks += taskName -> task
+    allTasksRegistered.countDown
+  }
+
+  def awaitTaskRegistered {
+    allTasksRegistered.await(60, TimeUnit.SECONDS)
+    assertEquals(0, allTasksRegistered.getCount)
+    assertEquals(totalTasks, tasks.size)
+    // Reset the registered latch, so we can use it again every time we start a new job.
+    TestTask.allTasksRegistered = new CountDownLatch(TestTask.totalTasks)
+  }
+}
+
+/**
+ * This class defines the base class for StreamTask used in integration test
+ * It implements some basic hooks for synchronization between the test class and the tasks
+ */
+abstract class TestTask extends StreamTask with InitableTask {
+  var received = ArrayBuffer[String]()
+  val initFinished = new CountDownLatch(1)
+  var gotMessage = new CountDownLatch(1)
+
+  def init(config: Config, context: TaskContext) {
+    TestTask.register(context.getTaskName, this)
+    testInit(config, context)
+    initFinished.countDown()
+  }
+
+  def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) {
+    val msg = envelope.getMessage.asInstanceOf[String]
+
+    System.err.println("TestTask.process(): %s" format msg)
+
+    received += msg
+
+    testProcess(envelope, collector, coordinator)
+
+    // Notify sender that we got a message.
+    gotMessage.countDown
+  }
+
+  def awaitMessage {
+    assertTrue("Timed out of waiting for message rather than received one.", gotMessage.await(60, TimeUnit.SECONDS))
+    assertEquals(0, gotMessage.getCount)
+    gotMessage = new CountDownLatch(1)
+  }
+
+  def testInit(config: Config, context: TaskContext)
+
+  def testProcess(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator)
+
+}

http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala
new file mode 100644
index 0000000..06a107b
--- /dev/null
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.test.integration
+
+import org.apache.samza.config.Config
+import org.apache.samza.storage.kv.KeyValueStore
+import org.apache.samza.system.IncomingMessageEnvelope
+import org.apache.samza.task.{MessageCollector, TaskContext, TaskCoordinator}
+import org.junit.Assert._
+import org.junit.{AfterClass, BeforeClass, Test}
+
+import scala.collection.JavaConversions._
+
+object TestShutdownStatefulTask {
+  val STORE_NAME = "loggedstore"
+  val STATE_TOPIC_STREAM = "storeChangelog"
+
+  @BeforeClass
+  def beforeSetupServers {
+    StreamTaskTestUtil.beforeSetupServers
+  }
+
+  @AfterClass
+  def afterCleanLogDirs {
+    StreamTaskTestUtil.afterCleanLogDirs
+  }
+}
+
+/**
+ * Test that does the following:
+ * 1. Start a single partition of TestShutdownTask using ThreadJobFactory.
+ * 2. Send six messages to input (1,2,3,2,99,99) and validate that all messages were received by TestShutdownStateStoreTask.
+ * 3. Shutdown the job.
+ * 4. Start the job again.
+ * 5. Validate that the job restored all states (1,2,3,99) to the store, including the pending flushed messages before shutdown
+ */
+class TestShutdownStatefulTask extends StreamTaskTestUtil {
+
+  StreamTaskTestUtil(Map(
+    "job.name" -> "state-stateful-world",
+    "task.class" -> "org.apache.samza.test.integration.ShutdownStateStoreTask",
+    "task.commit.ms" -> "-1",
+    "stores.loggedstore.factory" -> "org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory",
+    "stores.loggedstore.key.serde" -> "string",
+    "stores.loggedstore.msg.serde" -> "string",
+    "stores.loggedstore.changelog" -> "kafka.storeChangelog",
+    "stores.loggedstore.changelog.replication.factor" -> "1",
+    // However, don't have the inputs use the checkpoint manager
+    // since the second part of the test expects to replay the input streams.
+    "systems.kafka.streams.input.samza.reset.offset" -> "false"))
+
+  @Test
+  def testShouldStartAndRestore {
+    // Have to do this in one test to guarantee ordering.
+    testShouldStartTaskForFirstTime
+    testShouldRestoreStore
+  }
+
+  def testShouldStartTaskForFirstTime {
+    val (job, task) = startJob
+
+    // Validate that restored is empty.
+    assertEquals(0, task.initFinished.getCount)
+    assertEquals(0, task.asInstanceOf[ShutdownStateStoreTask].restored.size)
+    assertEquals(0, task.received.size)
+
+    // Send some messages to input stream.
+    send(task, "1")
+    send(task, "2")
+    send(task, "3")
+    send(task, "2")
+    send(task, "99")
+    send(task, "99")
+
+    stopJob(job)
+
+  }
+
+  def testShouldRestoreStore {
+    val (job, task) = startJob
+
+    // Validate that restored has expected data.
+    assertEquals(4, task.asInstanceOf[ShutdownStateStoreTask].restored.size)
+    assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("1").get.toInt == 1)
+    assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("2").get.toInt == 2)
+    assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("3").get.toInt == 1)
+    assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("99").get.toInt == 2)
+
+    stopJob(job)
+  }
+}
+
+/**
+ * This ShutdownStateStoreTask implements a simple task w/ a state store that counts the occurence of msgs received
+ */
+class ShutdownStateStoreTask extends TestTask {
+  var store: KeyValueStore[String, String] = null
+  var restored = scala.collection.mutable.Map[String, String]()
+
+  override def testInit(config: Config, context: TaskContext) {
+    store = context
+      .getStore(TestShutdownStatefulTask.STORE_NAME)
+      .asInstanceOf[KeyValueStore[String, String]]
+    val iter = store.all
+    iter.foreach( p => restored += (p.getKey -> p.getValue))
+    System.err.println("ShutdownStateStoreTask.init(): %s" format restored)
+    iter.close
+  }
+
+  override def testProcess(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) {
+    val msg = envelope.getMessage.asInstanceOf[String]
+
+    // Count the specific string received
+    val cntStr = store.get(msg)
+    if (cntStr == null) {
+      store.put(msg, "1")
+    } else {
+      val count = cntStr.toInt + 1
+      store.put(msg, count.toString)
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
index ea702a9..2240903 100644
--- a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
@@ -19,210 +19,56 @@
 
 package org.apache.samza.test.integration
 
-import java.util.Properties
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.TimeUnit
-import java.util
-
-import kafka.admin.AdminUtils
-import kafka.consumer.Consumer
-import kafka.consumer.ConsumerConfig
-import kafka.message.MessageAndMetadata
-import kafka.server.KafkaConfig
-import kafka.server.KafkaServer
-import kafka.utils.TestUtils
-import kafka.utils.TestZKUtils
-import kafka.utils.Utils
-import kafka.utils.ZKStringSerializer
-import kafka.zk.EmbeddedZookeeper
-import org.I0Itec.zkclient.ZkClient
-import org.apache.samza.Partition
-import org.apache.samza.checkpoint.Checkpoint
 import org.apache.samza.config.Config
-import org.apache.samza.job.local.ThreadJobFactory
-import org.apache.samza.config.MapConfig
-import org.apache.samza.container.TaskName
-import org.apache.samza.job.ApplicationStatus
-import org.apache.samza.job.StreamJob
 import org.apache.samza.storage.kv.KeyValueStore
-import org.apache.samza.system.kafka.TopicMetadataCache
-import org.apache.samza.system.{SystemStreamPartition, IncomingMessageEnvelope}
-import org.apache.samza.config.KafkaProducerConfig
-import org.apache.samza.task.InitableTask
-import org.apache.samza.task.MessageCollector
-import org.apache.samza.task.StreamTask
-import org.apache.samza.task.TaskContext
-import org.apache.samza.task.TaskCoordinator
+import org.apache.samza.system.IncomingMessageEnvelope
 import org.apache.samza.task.TaskCoordinator.RequestScope
-import org.apache.samza.util.ClientUtilTopicMetadataStore
-import org.apache.samza.util.TopicMetadataStore
-import org.apache.samza.job.JobRunner
+import org.apache.samza.task.{MessageCollector, TaskContext, TaskCoordinator}
 import org.junit.Assert._
-import org.junit.{BeforeClass, AfterClass, Test}
+import org.junit.{AfterClass, BeforeClass, Test}
+
 import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.mutable.HashMap
-import scala.collection.mutable.SynchronizedMap
-import org.apache.samza.job.JobRunner
-import org.apache.kafka.clients.producer.{ProducerConfig, Producer, ProducerRecord, KafkaProducer}
-import org.apache.samza.util.KafkaUtil
 
 object TestStatefulTask {
-  val INPUT_TOPIC = "input"
-  val STORE_NAME = "mystore"
-  val STATE_TOPIC_STREAM = "mystoreChangelog"
-  val TOTAL_TASK_NAMES = 1
-  val REPLICATION_FACTOR = 3
-
-  val zkConnect: String = TestZKUtils.zookeeperConnect
-  var zkClient: ZkClient = null
-  val zkConnectionTimeout = 6000
-  val zkSessionTimeout = 6000
-
-  val brokerId1 = 0
-  val brokerId2 = 1
-  val brokerId3 = 2
-  val ports = TestUtils.choosePorts(3)
-  val (port1, port2, port3) = (ports(0), ports(1), ports(2))
-
-  val props1 = TestUtils.createBrokerConfig(brokerId1, port1)
-  val props2 = TestUtils.createBrokerConfig(brokerId2, port2)
-  val props3 = TestUtils.createBrokerConfig(brokerId3, port3)
-  props1.setProperty("auto.create.topics.enable","false")
-  props2.setProperty("auto.create.topics.enable","false")
-  props3.setProperty("auto.create.topics.enable","false")
-
-  val config = new util.HashMap[String, Object]()
-  val brokers = "localhost:%d,localhost:%d,localhost:%d" format (port1, port2, port3)
-  config.put("bootstrap.servers", brokers)
-  config.put("request.required.acks", "-1")
-  config.put("serializer.class", "kafka.serializer.StringEncoder")
-  config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
-  config.put(ProducerConfig.RETRIES_CONFIG, (new Integer(Integer.MAX_VALUE-1)).toString())
-  val producerConfig = new KafkaProducerConfig("kafka", "i001", config)
-  var producer: Producer[Array[Byte], Array[Byte]] = null
-  val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123"))
-  val cp2 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "12345"))
-  var zookeeper: EmbeddedZookeeper = null
-  var server1: KafkaServer = null
-  var server2: KafkaServer = null
-  var server3: KafkaServer = null
-  var metadataStore: TopicMetadataStore = null
+    val STORE_NAME = "mystore"
+    val STATE_TOPIC_STREAM = "mystoreChangelog"
 
   @BeforeClass
   def beforeSetupServers {
-    zookeeper = new EmbeddedZookeeper(zkConnect)
-    server1 = TestUtils.createServer(new KafkaConfig(props1))
-    server2 = TestUtils.createServer(new KafkaConfig(props2))
-    server3 = TestUtils.createServer(new KafkaConfig(props3))
-    zkClient = new ZkClient(zkConnect + "/", 6000, 6000, ZKStringSerializer)
-    producer = new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties)
-    metadataStore = new ClientUtilTopicMetadataStore(brokers, "some-job-name")
-
-    createTopics
-    validateTopics
-  }
-
-  def createTopics {
-    AdminUtils.createTopic(
-      zkClient,
-      INPUT_TOPIC,
-      TOTAL_TASK_NAMES,
-      REPLICATION_FACTOR)
-  }
-
-  def validateTopics {
-    val topics = Set(INPUT_TOPIC)
-    var done = false
-    var retries = 0
-
-    while (!done && retries < 100) {
-      try {
-        val topicMetadataMap = TopicMetadataCache.getTopicMetadata(topics, "kafka", metadataStore.getTopicInfo)
-
-        topics.foreach(topic => {
-          val topicMetadata = topicMetadataMap(topic)
-          val errorCode = topicMetadata.errorCode
-
-          KafkaUtil.maybeThrowException(errorCode)
-        })
-
-        done = true
-      } catch {
-        case e: Exception =>
-          System.err.println("Got exception while validating test topics. Waiting and retrying.", e)
-          retries += 1
-          Thread.sleep(500)
-      }
-    }
-
-    if (retries >= 100) {
-      fail("Unable to successfully create topics. Tried to validate %s times." format retries)
-    }
+    StreamTaskTestUtil.beforeSetupServers
   }
 
   @AfterClass
   def afterCleanLogDirs {
-    producer.close()
-    server1.shutdown
-    server1.awaitShutdown()
-    server2.shutdown
-    server2.awaitShutdown()
-    server3.shutdown
-    server3.awaitShutdown()
-    Utils.rm(server1.config.logDirs)
-    Utils.rm(server2.config.logDirs)
-    Utils.rm(server3.config.logDirs)
-    zkClient.close
-    zookeeper.shutdown
+    StreamTaskTestUtil.afterCleanLogDirs
   }
 }
 
 /**
  * Test that does the following:
- *
- * 1. Starts ZK, and 3 kafka brokers.
- * 2. Create two topics: input and mystore.
- * 3. Validate that the topics were created successfully and have leaders.
- * 4. Start a single partition of TestTask using ThreadJobFactory.
- * 5. Send four messages to input (1,2,3,2), which contain one dupe (2).
- * 6. Validate that all messages were received by TestTask.
- * 7. Validate that TestTask called store.put() for all four messages, and that the messages ended up in the mystore topic.
- * 8. Kill the job.
- * 9. Start the job again.
- * 10. Validate that the job restored all messages (1,2,3) to the store.
- * 11. Send three more messages to input (4,5,5), and validate that TestTask receives them.
- * 12. Kill the job again.
+ * 1. Start a single partition of TestStateStoreTask using ThreadJobFactory.
+ * 2. Send four messages to input (1,2,3,2), which contain one dupe (2).
+ * 3. Validate that all messages were received by TestStateStoreTask.
+ * 4. Validate that TestStateStoreTask called store.put() for all four messages, and that the messages ended up in the mystore topic.
+ * 5. Kill the job.
+ * 6. Start the job again.
+ * 7. Validate that the job restored all messages (1,2,3) to the store.
+ * 8. Send three more messages to input (4,5,5), and validate that TestStateStoreTask receives them.
+ * 9. Kill the job again.
  */
-class TestStatefulTask {
-  import TestStatefulTask._
+class TestStatefulTask extends StreamTaskTestUtil {
 
-  val jobConfig = Map(
-    "job.factory.class" -> classOf[ThreadJobFactory].getCanonicalName,
+  StreamTaskTestUtil(Map(
     "job.name" -> "hello-stateful-world",
-    "task.class" -> "org.apache.samza.test.integration.TestTask",
-    "task.inputs" -> "kafka.input",
-    "serializers.registry.string.class" -> "org.apache.samza.serializers.StringSerdeFactory",
+    "task.class" -> "org.apache.samza.test.integration.StateStoreTestTask",
     "stores.mystore.factory" -> "org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory",
     "stores.mystore.key.serde" -> "string",
     "stores.mystore.msg.serde" -> "string",
     "stores.mystore.changelog" -> "kafka.mystoreChangelog",
     "stores.mystore.changelog.replication.factor" -> "1",
-    "systems.kafka.samza.factory" -> "org.apache.samza.system.kafka.KafkaSystemFactory",
-    // Always start consuming at offset 0. This avoids a race condition between
-    // the producer and the consumer in this test (SAMZA-166, SAMZA-224).
-    "systems.kafka.samza.offset.default" -> "oldest", // applies to a nonempty topic
-    "systems.kafka.consumer.auto.offset.reset" -> "smallest", // applies to an empty topic
-    "systems.kafka.samza.msg.serde" -> "string",
-    "systems.kafka.consumer.zookeeper.connect" -> zkConnect,
-    "systems.kafka.producer.bootstrap.servers" -> ("localhost:%s" format port1),
-    // Since using state, need a checkpoint manager
-    "task.checkpoint.factory" -> "org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory",
-    "task.checkpoint.system" -> "kafka",
-    "task.checkpoint.replication.factor" -> "1",
     // However, don't have the inputs use the checkpoint manager
     // since the second part of the test expects to replay the input streams.
-    "systems.kafka.streams.input.samza.reset.offset" -> "true")
+    "systems.kafka.streams.input.samza.reset.offset" -> "true"))
 
   @Test
   def testShouldStartAndRestore {
@@ -236,7 +82,7 @@ class TestStatefulTask {
 
     // Validate that restored is empty.
     assertEquals(0, task.initFinished.getCount)
-    assertEquals(0, task.restored.size)
+    assertEquals(0, task.asInstanceOf[StateStoreTestTask].restored.size)
     assertEquals(0, task.received.size)
 
     // Send some messages to input stream.
@@ -248,7 +94,7 @@ class TestStatefulTask {
     send(task, "-99")
 
     // Validate that messages appear in store stream.
-    val messages = readAll(STATE_TOPIC_STREAM, 5, "testShouldStartTaskForFirstTime")
+    val messages = readAll(TestStatefulTask.STATE_TOPIC_STREAM, 5, "testShouldStartTaskForFirstTime")
 
     assertEquals(6, messages.length)
     assertEquals("1", messages(0))
@@ -265,10 +111,10 @@ class TestStatefulTask {
     val (job, task) = startJob
 
     // Validate that restored has expected data.
-    assertEquals(3, task.restored.size)
-    assertTrue(task.restored.contains("1"))
-    assertTrue(task.restored.contains("2"))
-    assertTrue(task.restored.contains("3"))
+    assertEquals(3, task.asInstanceOf[StateStoreTestTask].restored.size)
+    assertTrue(task.asInstanceOf[StateStoreTestTask].restored.contains("1"))
+    assertTrue(task.asInstanceOf[StateStoreTestTask].restored.contains("2"))
+    assertTrue(task.asInstanceOf[StateStoreTestTask].restored.contains("3"))
 
     var count = 0
 
@@ -290,7 +136,7 @@ class TestStatefulTask {
     send(task, "5")
 
     // Validate that messages appear in store stream.
-    val messages = readAll(STATE_TOPIC_STREAM, 14, "testShouldRestoreStore")
+    val messages = readAll(TestStatefulTask.STATE_TOPIC_STREAM, 14, "testShouldRestoreStore")
 
     assertEquals(15, messages.length)
     // From initial start.
@@ -315,126 +161,25 @@ class TestStatefulTask {
     stopJob(job)
   }
 
-  /**
-   * Start a job for TestJob, and do some basic sanity checks around startup
-   * time, number of partitions, etc.
-   */
-  def startJob = {
-    // Start task.
-    val job = new JobRunner(new MapConfig(jobConfig)).run
-    assertEquals(ApplicationStatus.Running, job.waitForStatus(ApplicationStatus.Running, 60000))
-    TestTask.awaitTaskRegistered
-    val tasks = TestTask.tasks
-    assertEquals("Should only have a single partition in this task", 1, tasks.size)
-    val task = tasks.values.toList.head
-    task.initFinished.await(60, TimeUnit.SECONDS)
-    assertEquals(0, task.initFinished.getCount)
-    (job, task)
-  }
-
-  /**
-   * Kill a job, and wait for an unsuccessful finish (since this throws an
-   * interrupt, which is forwarded on to ThreadJob, and marked as a failure).
-   */
-  def stopJob(job: StreamJob) {
-    // Shutdown task.
-    job.kill
-    assertEquals(ApplicationStatus.UnsuccessfulFinish, job.waitForFinish(60000))
-  }
-
-  /**
-   * Send a message to the input topic, and validate that it gets to the test task.
-   */
-  def send(task: TestTask, msg: String) {
-    producer.send(new ProducerRecord(INPUT_TOPIC, msg.getBytes)).get()
-    task.awaitMessage
-    assertEquals(msg, task.received.last)
-  }
-
-  /**
-   * Read all messages from a topic starting from last saved offset for group.
-   * To read all from offset 0, specify a unique, new group string.
-   */
-  def readAll(topic: String, maxOffsetInclusive: Int, group: String): List[String] = {
-    val props = new Properties
-
-    props.put("zookeeper.connect", zkConnect)
-    props.put("group.id", group)
-    props.put("auto.offset.reset", "smallest")
-
-    val consumerConfig = new ConsumerConfig(props)
-    val consumerConnector = Consumer.create(consumerConfig)
-    var stream = consumerConnector.createMessageStreams(Map(topic -> 1)).get(topic).get.get(0).iterator
-    var message: MessageAndMetadata[Array[Byte], Array[Byte]] = null
-    var messages = ArrayBuffer[String]()
-
-    while (message == null || message.offset < maxOffsetInclusive) {
-      message = stream.next
-      if (message.message == null) {
-        messages += null
-      } else {
-        messages += new String(message.message, "UTF-8")
-      }
-      System.err.println("TestStatefulTask.readAll(): offset=%s, message=%s" format (message.offset, messages.last))
-    }
-
-    consumerConnector.shutdown
-
-    messages.toList
-  }
-}
-
-object TestTask {
-  val tasks = new HashMap[TaskName, TestTask] with SynchronizedMap[TaskName, TestTask]
-  @volatile var allTasksRegistered = new CountDownLatch(TestStatefulTask.TOTAL_TASK_NAMES)
-
-  /**
-   * Static method that tasks can use to register themselves with. Useful so
-   * we don't have to sneak into the ThreadJob/SamzaContainer to get our test
-   * tasks.
-   */
-  def register(taskName: TaskName, task: TestTask) {
-    tasks += taskName -> task
-    allTasksRegistered.countDown
-  }
-
-  def awaitTaskRegistered {
-    allTasksRegistered.await(60, TimeUnit.SECONDS)
-    assertEquals(0, allTasksRegistered.getCount)
-    assertEquals(TestStatefulTask.TOTAL_TASK_NAMES, tasks.size)
-    // Reset the registered latch, so we can use it again every time we start a new job.
-    TestTask.allTasksRegistered = new CountDownLatch(TestStatefulTask.TOTAL_TASK_NAMES)
-  }
 }
 
-class TestTask extends StreamTask with InitableTask {
+class StateStoreTestTask extends TestTask {
   var store: KeyValueStore[String, String] = null
   var restored = Set[String]()
-  var received = ArrayBuffer[String]()
-  val initFinished = new CountDownLatch(1)
-  var gotMessage = new CountDownLatch(1)
-
-  def init(config: Config, context: TaskContext) {
-    TestTask.register(context.getTaskName, this)
-    store = context
-      .getStore(TestStatefulTask.STORE_NAME)
-      .asInstanceOf[KeyValueStore[String, String]]
+
+  override def testInit(config: Config, context: TaskContext): Unit = {
+    store = context.getStore(TestStatefulTask.STORE_NAME).asInstanceOf[KeyValueStore[String, String]]
     val iter = store.all
     restored ++= iter
       .map(_.getValue)
       .toSet
-    System.err.println("TestTask.init(): %s" format restored)
+    System.err.println("StateStoreTestTask.init(): %s" format restored)
     iter.close
-    initFinished.countDown()
   }
 
-  def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) {
+  override def testProcess(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator): Unit = {
     val msg = envelope.getMessage.asInstanceOf[String]
 
-    System.err.println("TestTask.process(): %s" format msg)
-
-    received += msg
-
     // A negative string means delete
     if (msg.startsWith("-")) {
       store.delete(msg.substring(1))
@@ -443,14 +188,6 @@ class TestTask extends StreamTask with InitableTask {
     }
 
     coordinator.commit(RequestScope.ALL_TASKS_IN_CONTAINER)
-
-    // Notify sender that we got a message.
-    gotMessage.countDown
-  }
-
-  def awaitMessage {
-    assertTrue("Timed out of waiting for message rather than received one.", gotMessage.await(60, TimeUnit.SECONDS))
-    assertEquals(0, gotMessage.getCount)
-    gotMessage = new CountDownLatch(1)
   }
 }
+