You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by ni...@apache.org on 2015/08/08 02:05:22 UTC
samza git commit: SAMZA-744: Fix the container shutdown sequence s.t.
KV-stores are flushed before shutting down producers
Repository: samza
Updated Branches:
refs/heads/master c32022498 -> 98df876dd
SAMZA-744: Fix the container shutdown sequence s.t. KV-stores are flushed before shutting down producers
Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/98df876d
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/98df876d
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/98df876d
Branch: refs/heads/master
Commit: 98df876dd4492b7d32422de0eccff1cbc26e1888
Parents: c320224
Author: Yi Pan <ni...@gmail.com>
Authored: Fri Aug 7 17:05:08 2015 -0700
Committer: Yi Pan (Data Infrastructure) <yi...@linkedin.com>
Committed: Fri Aug 7 17:05:08 2015 -0700
----------------------------------------------------------------------
build.gradle | 2 +-
.../apache/samza/container/SamzaContainer.scala | 2 +-
.../test/integration/StreamTaskTestUtil.scala | 335 +++++++++++++++++++
.../integration/TestShutdownStatefulTask.scala | 139 ++++++++
.../test/integration/TestStatefulTask.scala | 333 ++----------------
5 files changed, 511 insertions(+), 300 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index cc30b45..a935088 100644
--- a/build.gradle
+++ b/build.gradle
@@ -78,7 +78,7 @@ rat {
'gradle/wrapper/**',
'gradlew',
'gradlew.bat',
- 'samza-test/state/mystore/**',
+ 'samza-test/state/**',
'README.md',
'RELEASE.md',
'samza-test/src/main/resources/**',
http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
----------------------------------------------------------------------
diff --git a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
index 702ce26..85b012b 100644
--- a/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
+++ b/samza-core/src/main/scala/org/apache/samza/container/SamzaContainer.scala
@@ -581,8 +581,8 @@ class SamzaContainer(
shutdownConsumers
shutdownTask
- shutdownProducers
shutdownStores
+ shutdownProducers
shutdownLocalityManager
shutdownOffsetManager
shutdownMetrics
http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
new file mode 100644
index 0000000..c6e994f
--- /dev/null
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/StreamTaskTestUtil.scala
@@ -0,0 +1,335 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.test.integration
+
+import java.util
+import java.util.Properties
+import java.util.concurrent.{CountDownLatch, TimeUnit}
+
+import kafka.admin.AdminUtils
+import kafka.consumer.{Consumer, ConsumerConfig}
+import kafka.message.MessageAndMetadata
+import kafka.server.{KafkaConfig, KafkaServer}
+import kafka.utils.{TestUtils, TestZKUtils, Utils, ZKStringSerializer}
+import kafka.zk.EmbeddedZookeeper
+import org.I0Itec.zkclient.ZkClient
+import org.apache.kafka.clients.producer.{KafkaProducer, Producer, ProducerConfig, ProducerRecord}
+import org.apache.samza.Partition
+import org.apache.samza.checkpoint.Checkpoint
+import org.apache.samza.config.{Config, KafkaProducerConfig, MapConfig}
+import org.apache.samza.container.TaskName
+import org.apache.samza.job.local.ThreadJobFactory
+import org.apache.samza.job.{ApplicationStatus, JobRunner, StreamJob}
+import org.apache.samza.system.kafka.TopicMetadataCache
+import org.apache.samza.system.{IncomingMessageEnvelope, SystemStreamPartition}
+import org.apache.samza.task._
+import org.apache.samza.util.{ClientUtilTopicMetadataStore, KafkaUtil, TopicMetadataStore}
+import org.junit.Assert._
+
+import scala.collection.JavaConversions._
+import scala.collection.mutable.{ArrayBuffer, HashMap, SynchronizedMap}
+
+/*
+ * This creates an singleton instance of TestBaseStreamTask and implement the helper functions to
+ * 1. start the local ZooKeeper server
+ * 2. start the local Kafka brokers
+ * 3. create and validate test topics
+ * 4. shutdown servers and cleanup test directories and files
+ */
+object StreamTaskTestUtil {
+ val INPUT_TOPIC = "input"
+ val TOTAL_TASK_NAMES = 1
+ val REPLICATION_FACTOR = 3
+
+ val zkConnect: String = TestZKUtils.zookeeperConnect
+ var zkClient: ZkClient = null
+ val zkConnectionTimeout = 6000
+ val zkSessionTimeout = 6000
+
+ val brokerId1 = 0
+ val brokerId2 = 1
+ val brokerId3 = 2
+ val ports = TestUtils.choosePorts(3)
+ val (port1, port2, port3) = (ports(0), ports(1), ports(2))
+
+ val props1 = TestUtils.createBrokerConfig(brokerId1, port1)
+ val props2 = TestUtils.createBrokerConfig(brokerId2, port2)
+ val props3 = TestUtils.createBrokerConfig(brokerId3, port3)
+ props1.setProperty("auto.create.topics.enable","false")
+ props2.setProperty("auto.create.topics.enable","false")
+ props3.setProperty("auto.create.topics.enable","false")
+
+ val config = new util.HashMap[String, Object]()
+ val brokers = "localhost:%d,localhost:%d,localhost:%d" format (port1, port2, port3)
+ config.put("bootstrap.servers", brokers)
+ config.put("request.required.acks", "-1")
+ config.put("serializer.class", "kafka.serializer.StringEncoder")
+ config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
+ config.put(ProducerConfig.RETRIES_CONFIG, (new Integer(Integer.MAX_VALUE-1)).toString())
+ val producerConfig = new KafkaProducerConfig("kafka", "i001", config)
+ var producer: Producer[Array[Byte], Array[Byte]] = null
+ val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123"))
+ val cp2 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "12345"))
+ var zookeeper: EmbeddedZookeeper = null
+ var server1: KafkaServer = null
+ var server2: KafkaServer = null
+ var server3: KafkaServer = null
+ var metadataStore: TopicMetadataStore = null
+
+ /*
+ * This is the default job configuration. Each test class can override the default configuration below.
+ */
+ var jobConfig = Map(
+ "job.factory.class" -> classOf[ThreadJobFactory].getCanonicalName,
+ "task.inputs" -> "kafka.input",
+ "serializers.registry.string.class" -> "org.apache.samza.serializers.StringSerdeFactory",
+ "systems.kafka.samza.factory" -> "org.apache.samza.system.kafka.KafkaSystemFactory",
+ // Always start consuming at offset 0. This avoids a race condition between
+ // the producer and the consumer in this test (SAMZA-166, SAMZA-224).
+ "systems.kafka.samza.offset.default" -> "oldest", // applies to a nonempty topic
+ "systems.kafka.consumer.auto.offset.reset" -> "smallest", // applies to an empty topic
+ "systems.kafka.samza.msg.serde" -> "string",
+ "systems.kafka.consumer.zookeeper.connect" -> zkConnect,
+ "systems.kafka.producer.bootstrap.servers" -> ("localhost:%s" format port1),
+ // Since using state, need a checkpoint manager
+ // Due to SAMZA-754, the following section can not be removed yet.
+ "task.checkpoint.factory" -> "org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory",
+ "task.checkpoint.system" -> "kafka",
+ "task.checkpoint.replication.factor" -> "1",
+ // However, don't have the inputs use the checkpoint manager
+ // since the second part of the test expects to replay the input streams.
+ "systems.kafka.streams.input.samza.reset.offset" -> "false")
+
+ def apply(map: Map[String, String]): Unit = {
+ jobConfig ++= map
+ TestTask.reset()
+ }
+
+ def beforeSetupServers {
+ zookeeper = new EmbeddedZookeeper(zkConnect)
+ server1 = TestUtils.createServer(new KafkaConfig(props1))
+ server2 = TestUtils.createServer(new KafkaConfig(props2))
+ server3 = TestUtils.createServer(new KafkaConfig(props3))
+ zkClient = new ZkClient(zkConnect + "/", 6000, 6000, ZKStringSerializer)
+ producer = new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties)
+ metadataStore = new ClientUtilTopicMetadataStore(brokers, "some-job-name")
+
+ createTopics
+ validateTopics
+ }
+
+ def createTopics {
+ AdminUtils.createTopic(
+ zkClient,
+ INPUT_TOPIC,
+ TOTAL_TASK_NAMES,
+ REPLICATION_FACTOR)
+ }
+
+ def validateTopics {
+ val topics = Set(INPUT_TOPIC)
+ var done = false
+ var retries = 0
+
+ while (!done && retries < 100) {
+ try {
+ val topicMetadataMap = TopicMetadataCache.getTopicMetadata(topics, "kafka", metadataStore.getTopicInfo)
+
+ topics.foreach(topic => {
+ val topicMetadata = topicMetadataMap(topic)
+ val errorCode = topicMetadata.errorCode
+
+ KafkaUtil.maybeThrowException(errorCode)
+ })
+
+ done = true
+ } catch {
+ case e: Exception =>
+ System.err.println("Got exception while validating test topics. Waiting and retrying.", e)
+ retries += 1
+ Thread.sleep(500)
+ }
+ }
+
+ if (retries >= 100) {
+ fail("Unable to successfully create topics. Tried to validate %s times." format retries)
+ }
+ }
+
+ def afterCleanLogDirs {
+ producer.close()
+ server1.shutdown
+ server1.awaitShutdown()
+ server2.shutdown
+ server2.awaitShutdown()
+ server3.shutdown
+ server3.awaitShutdown()
+ Utils.rm(server1.config.logDirs)
+ Utils.rm(server2.config.logDirs)
+ Utils.rm(server3.config.logDirs)
+ zkClient.close
+ zookeeper.shutdown
+ }
+}
+
+/* This class implement the base utility to implement an integration test for StreamTask
+ * It implements helper functions to start/stop the job, send messages to a task, and read all messages from a topic
+ */
+class StreamTaskTestUtil {
+ import StreamTaskTestUtil._
+
+ /**
+ * Start a job for TestTask, and do some basic sanity checks around startup
+ * time, number of partitions, etc.
+ */
+ def startJob = {
+ // Start task.
+ val job = new JobRunner(new MapConfig(jobConfig)).run
+ assertEquals(ApplicationStatus.Running, job.waitForStatus(ApplicationStatus.Running, 60000))
+ TestTask.awaitTaskRegistered
+ val tasks = TestTask.tasks
+ assertEquals("Should only have a single partition in this task", 1, tasks.size)
+ val task = tasks.values.toList.head
+ task.initFinished.await(60, TimeUnit.SECONDS)
+ assertEquals(0, task.initFinished.getCount)
+ (job, task)
+ }
+
+ /**
+ * Kill a job, and wait for an unsuccessful finish (since this throws an
+ * interrupt, which is forwarded on to ThreadJob, and marked as a failure).
+ */
+ def stopJob(job: StreamJob) {
+ // Shutdown task.
+ job.kill
+ assertEquals(ApplicationStatus.UnsuccessfulFinish, job.waitForFinish(60000))
+ }
+
+ /**
+ * Send a message to the input topic, and validate that it gets to the test task.
+ */
+ def send(task: TestTask, msg: String) {
+ producer.send(new ProducerRecord(INPUT_TOPIC, msg.getBytes)).get()
+ task.awaitMessage
+ assertEquals(msg, task.received.last)
+ }
+
+ /**
+ * Read all messages from a topic starting from last saved offset for group.
+ * To read all from offset 0, specify a unique, new group string.
+ */
+ def readAll(topic: String, maxOffsetInclusive: Int, group: String): List[String] = {
+ val props = new Properties
+
+ props.put("zookeeper.connect", zkConnect)
+ props.put("group.id", group)
+ props.put("auto.offset.reset", "smallest")
+
+ val consumerConfig = new ConsumerConfig(props)
+ val consumerConnector = Consumer.create(consumerConfig)
+ var stream = consumerConnector.createMessageStreams(Map(topic -> 1)).get(topic).get.get(0).iterator
+ var message: MessageAndMetadata[Array[Byte], Array[Byte]] = null
+ var messages = ArrayBuffer[String]()
+
+ while (message == null || message.offset < maxOffsetInclusive) {
+ message = stream.next
+ if (message.message == null) {
+ messages += null
+ } else {
+ messages += new String(message.message, "UTF-8")
+ }
+ System.err.println("StreamTaskTestUtil.readAll(): offset=%s, message=%s" format (message.offset, messages.last))
+ }
+
+ consumerConnector.shutdown
+
+ messages.toList
+ }
+
+}
+
+object TestTask {
+ val tasks = new HashMap[TaskName, TestTask] with SynchronizedMap[TaskName, TestTask]
+ var totalTasks = 1
+ @volatile var allTasksRegistered = new CountDownLatch(totalTasks)
+
+ def reset(): Unit = {
+ TestTask.totalTasks = StreamTaskTestUtil.TOTAL_TASK_NAMES
+ TestTask.allTasksRegistered = new CountDownLatch(TestTask.totalTasks)
+ }
+
+ /**
+ * Static method that tasks can use to register themselves with. Useful so
+ * we don't have to sneak into the ThreadJob/SamzaContainer to get our test
+ * tasks.
+ */
+ def register(taskName: TaskName, task: TestTask) {
+ tasks += taskName -> task
+ allTasksRegistered.countDown
+ }
+
+ def awaitTaskRegistered {
+ allTasksRegistered.await(60, TimeUnit.SECONDS)
+ assertEquals(0, allTasksRegistered.getCount)
+ assertEquals(totalTasks, tasks.size)
+ // Reset the registered latch, so we can use it again every time we start a new job.
+ TestTask.allTasksRegistered = new CountDownLatch(TestTask.totalTasks)
+ }
+}
+
+/**
+ * This class defines the base class for StreamTask used in integration test
+ * It implements some basic hooks for synchronization between the test class and the tasks
+ */
+abstract class TestTask extends StreamTask with InitableTask {
+ var received = ArrayBuffer[String]()
+ val initFinished = new CountDownLatch(1)
+ var gotMessage = new CountDownLatch(1)
+
+ def init(config: Config, context: TaskContext) {
+ TestTask.register(context.getTaskName, this)
+ testInit(config, context)
+ initFinished.countDown()
+ }
+
+ def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) {
+ val msg = envelope.getMessage.asInstanceOf[String]
+
+ System.err.println("TestTask.process(): %s" format msg)
+
+ received += msg
+
+ testProcess(envelope, collector, coordinator)
+
+ // Notify sender that we got a message.
+ gotMessage.countDown
+ }
+
+ def awaitMessage {
+ assertTrue("Timed out of waiting for message rather than received one.", gotMessage.await(60, TimeUnit.SECONDS))
+ assertEquals(0, gotMessage.getCount)
+ gotMessage = new CountDownLatch(1)
+ }
+
+ def testInit(config: Config, context: TaskContext)
+
+ def testProcess(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator)
+
+}
http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala
new file mode 100644
index 0000000..06a107b
--- /dev/null
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/TestShutdownStatefulTask.scala
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.samza.test.integration
+
+import org.apache.samza.config.Config
+import org.apache.samza.storage.kv.KeyValueStore
+import org.apache.samza.system.IncomingMessageEnvelope
+import org.apache.samza.task.{MessageCollector, TaskContext, TaskCoordinator}
+import org.junit.Assert._
+import org.junit.{AfterClass, BeforeClass, Test}
+
+import scala.collection.JavaConversions._
+
+object TestShutdownStatefulTask {
+ val STORE_NAME = "loggedstore"
+ val STATE_TOPIC_STREAM = "storeChangelog"
+
+ @BeforeClass
+ def beforeSetupServers {
+ StreamTaskTestUtil.beforeSetupServers
+ }
+
+ @AfterClass
+ def afterCleanLogDirs {
+ StreamTaskTestUtil.afterCleanLogDirs
+ }
+}
+
+/**
+ * Test that does the following:
+ * 1. Start a single partition of TestShutdownTask using ThreadJobFactory.
+ * 2. Send six messages to input (1,2,3,2,99,99) and validate that all messages were received by TestShutdownStateStoreTask.
+ * 3. Shutdown the job.
+ * 4. Start the job again.
+ * 5. Validate that the job restored all states (1,2,3,99) to the store, including the pending flushed messages before shutdown
+ */
+class TestShutdownStatefulTask extends StreamTaskTestUtil {
+
+ StreamTaskTestUtil(Map(
+ "job.name" -> "state-stateful-world",
+ "task.class" -> "org.apache.samza.test.integration.ShutdownStateStoreTask",
+ "task.commit.ms" -> "-1",
+ "stores.loggedstore.factory" -> "org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory",
+ "stores.loggedstore.key.serde" -> "string",
+ "stores.loggedstore.msg.serde" -> "string",
+ "stores.loggedstore.changelog" -> "kafka.storeChangelog",
+ "stores.loggedstore.changelog.replication.factor" -> "1",
+ // However, don't have the inputs use the checkpoint manager
+ // since the second part of the test expects to replay the input streams.
+ "systems.kafka.streams.input.samza.reset.offset" -> "false"))
+
+ @Test
+ def testShouldStartAndRestore {
+ // Have to do this in one test to guarantee ordering.
+ testShouldStartTaskForFirstTime
+ testShouldRestoreStore
+ }
+
+ def testShouldStartTaskForFirstTime {
+ val (job, task) = startJob
+
+ // Validate that restored is empty.
+ assertEquals(0, task.initFinished.getCount)
+ assertEquals(0, task.asInstanceOf[ShutdownStateStoreTask].restored.size)
+ assertEquals(0, task.received.size)
+
+ // Send some messages to input stream.
+ send(task, "1")
+ send(task, "2")
+ send(task, "3")
+ send(task, "2")
+ send(task, "99")
+ send(task, "99")
+
+ stopJob(job)
+
+ }
+
+ def testShouldRestoreStore {
+ val (job, task) = startJob
+
+ // Validate that restored has expected data.
+ assertEquals(4, task.asInstanceOf[ShutdownStateStoreTask].restored.size)
+ assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("1").get.toInt == 1)
+ assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("2").get.toInt == 2)
+ assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("3").get.toInt == 1)
+ assertTrue(task.asInstanceOf[ShutdownStateStoreTask].restored.get("99").get.toInt == 2)
+
+ stopJob(job)
+ }
+}
+
+/**
+ * This ShutdownStateStoreTask implements a simple task w/ a state store that counts the occurence of msgs received
+ */
+class ShutdownStateStoreTask extends TestTask {
+ var store: KeyValueStore[String, String] = null
+ var restored = scala.collection.mutable.Map[String, String]()
+
+ override def testInit(config: Config, context: TaskContext) {
+ store = context
+ .getStore(TestShutdownStatefulTask.STORE_NAME)
+ .asInstanceOf[KeyValueStore[String, String]]
+ val iter = store.all
+ iter.foreach( p => restored += (p.getKey -> p.getValue))
+ System.err.println("ShutdownStateStoreTask.init(): %s" format restored)
+ iter.close
+ }
+
+ override def testProcess(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) {
+ val msg = envelope.getMessage.asInstanceOf[String]
+
+ // Count the specific string received
+ val cntStr = store.get(msg)
+ if (cntStr == null) {
+ store.put(msg, "1")
+ } else {
+ val count = cntStr.toInt + 1
+ store.put(msg, count.toString)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/samza/blob/98df876d/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
----------------------------------------------------------------------
diff --git a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
index ea702a9..2240903 100644
--- a/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
+++ b/samza-test/src/test/scala/org/apache/samza/test/integration/TestStatefulTask.scala
@@ -19,210 +19,56 @@
package org.apache.samza.test.integration
-import java.util.Properties
-import java.util.concurrent.CountDownLatch
-import java.util.concurrent.TimeUnit
-import java.util
-
-import kafka.admin.AdminUtils
-import kafka.consumer.Consumer
-import kafka.consumer.ConsumerConfig
-import kafka.message.MessageAndMetadata
-import kafka.server.KafkaConfig
-import kafka.server.KafkaServer
-import kafka.utils.TestUtils
-import kafka.utils.TestZKUtils
-import kafka.utils.Utils
-import kafka.utils.ZKStringSerializer
-import kafka.zk.EmbeddedZookeeper
-import org.I0Itec.zkclient.ZkClient
-import org.apache.samza.Partition
-import org.apache.samza.checkpoint.Checkpoint
import org.apache.samza.config.Config
-import org.apache.samza.job.local.ThreadJobFactory
-import org.apache.samza.config.MapConfig
-import org.apache.samza.container.TaskName
-import org.apache.samza.job.ApplicationStatus
-import org.apache.samza.job.StreamJob
import org.apache.samza.storage.kv.KeyValueStore
-import org.apache.samza.system.kafka.TopicMetadataCache
-import org.apache.samza.system.{SystemStreamPartition, IncomingMessageEnvelope}
-import org.apache.samza.config.KafkaProducerConfig
-import org.apache.samza.task.InitableTask
-import org.apache.samza.task.MessageCollector
-import org.apache.samza.task.StreamTask
-import org.apache.samza.task.TaskContext
-import org.apache.samza.task.TaskCoordinator
+import org.apache.samza.system.IncomingMessageEnvelope
import org.apache.samza.task.TaskCoordinator.RequestScope
-import org.apache.samza.util.ClientUtilTopicMetadataStore
-import org.apache.samza.util.TopicMetadataStore
-import org.apache.samza.job.JobRunner
+import org.apache.samza.task.{MessageCollector, TaskContext, TaskCoordinator}
import org.junit.Assert._
-import org.junit.{BeforeClass, AfterClass, Test}
+import org.junit.{AfterClass, BeforeClass, Test}
+
import scala.collection.JavaConversions._
-import scala.collection.mutable.ArrayBuffer
-import scala.collection.mutable.HashMap
-import scala.collection.mutable.SynchronizedMap
-import org.apache.samza.job.JobRunner
-import org.apache.kafka.clients.producer.{ProducerConfig, Producer, ProducerRecord, KafkaProducer}
-import org.apache.samza.util.KafkaUtil
object TestStatefulTask {
- val INPUT_TOPIC = "input"
- val STORE_NAME = "mystore"
- val STATE_TOPIC_STREAM = "mystoreChangelog"
- val TOTAL_TASK_NAMES = 1
- val REPLICATION_FACTOR = 3
-
- val zkConnect: String = TestZKUtils.zookeeperConnect
- var zkClient: ZkClient = null
- val zkConnectionTimeout = 6000
- val zkSessionTimeout = 6000
-
- val brokerId1 = 0
- val brokerId2 = 1
- val brokerId3 = 2
- val ports = TestUtils.choosePorts(3)
- val (port1, port2, port3) = (ports(0), ports(1), ports(2))
-
- val props1 = TestUtils.createBrokerConfig(brokerId1, port1)
- val props2 = TestUtils.createBrokerConfig(brokerId2, port2)
- val props3 = TestUtils.createBrokerConfig(brokerId3, port3)
- props1.setProperty("auto.create.topics.enable","false")
- props2.setProperty("auto.create.topics.enable","false")
- props3.setProperty("auto.create.topics.enable","false")
-
- val config = new util.HashMap[String, Object]()
- val brokers = "localhost:%d,localhost:%d,localhost:%d" format (port1, port2, port3)
- config.put("bootstrap.servers", brokers)
- config.put("request.required.acks", "-1")
- config.put("serializer.class", "kafka.serializer.StringEncoder")
- config.put(ProducerConfig.MAX_IN_FLIGHT_REQUESTS_PER_CONNECTION, "1")
- config.put(ProducerConfig.RETRIES_CONFIG, (new Integer(Integer.MAX_VALUE-1)).toString())
- val producerConfig = new KafkaProducerConfig("kafka", "i001", config)
- var producer: Producer[Array[Byte], Array[Byte]] = null
- val cp1 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "123"))
- val cp2 = new Checkpoint(Map(new SystemStreamPartition("kafka", "topic", new Partition(0)) -> "12345"))
- var zookeeper: EmbeddedZookeeper = null
- var server1: KafkaServer = null
- var server2: KafkaServer = null
- var server3: KafkaServer = null
- var metadataStore: TopicMetadataStore = null
+ val STORE_NAME = "mystore"
+ val STATE_TOPIC_STREAM = "mystoreChangelog"
@BeforeClass
def beforeSetupServers {
- zookeeper = new EmbeddedZookeeper(zkConnect)
- server1 = TestUtils.createServer(new KafkaConfig(props1))
- server2 = TestUtils.createServer(new KafkaConfig(props2))
- server3 = TestUtils.createServer(new KafkaConfig(props3))
- zkClient = new ZkClient(zkConnect + "/", 6000, 6000, ZKStringSerializer)
- producer = new KafkaProducer[Array[Byte], Array[Byte]](producerConfig.getProducerProperties)
- metadataStore = new ClientUtilTopicMetadataStore(brokers, "some-job-name")
-
- createTopics
- validateTopics
- }
-
- def createTopics {
- AdminUtils.createTopic(
- zkClient,
- INPUT_TOPIC,
- TOTAL_TASK_NAMES,
- REPLICATION_FACTOR)
- }
-
- def validateTopics {
- val topics = Set(INPUT_TOPIC)
- var done = false
- var retries = 0
-
- while (!done && retries < 100) {
- try {
- val topicMetadataMap = TopicMetadataCache.getTopicMetadata(topics, "kafka", metadataStore.getTopicInfo)
-
- topics.foreach(topic => {
- val topicMetadata = topicMetadataMap(topic)
- val errorCode = topicMetadata.errorCode
-
- KafkaUtil.maybeThrowException(errorCode)
- })
-
- done = true
- } catch {
- case e: Exception =>
- System.err.println("Got exception while validating test topics. Waiting and retrying.", e)
- retries += 1
- Thread.sleep(500)
- }
- }
-
- if (retries >= 100) {
- fail("Unable to successfully create topics. Tried to validate %s times." format retries)
- }
+ StreamTaskTestUtil.beforeSetupServers
}
@AfterClass
def afterCleanLogDirs {
- producer.close()
- server1.shutdown
- server1.awaitShutdown()
- server2.shutdown
- server2.awaitShutdown()
- server3.shutdown
- server3.awaitShutdown()
- Utils.rm(server1.config.logDirs)
- Utils.rm(server2.config.logDirs)
- Utils.rm(server3.config.logDirs)
- zkClient.close
- zookeeper.shutdown
+ StreamTaskTestUtil.afterCleanLogDirs
}
}
/**
* Test that does the following:
- *
- * 1. Starts ZK, and 3 kafka brokers.
- * 2. Create two topics: input and mystore.
- * 3. Validate that the topics were created successfully and have leaders.
- * 4. Start a single partition of TestTask using ThreadJobFactory.
- * 5. Send four messages to input (1,2,3,2), which contain one dupe (2).
- * 6. Validate that all messages were received by TestTask.
- * 7. Validate that TestTask called store.put() for all four messages, and that the messages ended up in the mystore topic.
- * 8. Kill the job.
- * 9. Start the job again.
- * 10. Validate that the job restored all messages (1,2,3) to the store.
- * 11. Send three more messages to input (4,5,5), and validate that TestTask receives them.
- * 12. Kill the job again.
+ * 1. Start a single partition of TestStateStoreTask using ThreadJobFactory.
+ * 2. Send four messages to input (1,2,3,2), which contain one dupe (2).
+ * 3. Validate that all messages were received by TestStateStoreTask.
+ * 4. Validate that TestStateStoreTask called store.put() for all four messages, and that the messages ended up in the mystore topic.
+ * 5. Kill the job.
+ * 6. Start the job again.
+ * 7. Validate that the job restored all messages (1,2,3) to the store.
+ * 8. Send three more messages to input (4,5,5), and validate that TestStateStoreTask receives them.
+ * 9. Kill the job again.
*/
-class TestStatefulTask {
- import TestStatefulTask._
+class TestStatefulTask extends StreamTaskTestUtil {
- val jobConfig = Map(
- "job.factory.class" -> classOf[ThreadJobFactory].getCanonicalName,
+ StreamTaskTestUtil(Map(
"job.name" -> "hello-stateful-world",
- "task.class" -> "org.apache.samza.test.integration.TestTask",
- "task.inputs" -> "kafka.input",
- "serializers.registry.string.class" -> "org.apache.samza.serializers.StringSerdeFactory",
+ "task.class" -> "org.apache.samza.test.integration.StateStoreTestTask",
"stores.mystore.factory" -> "org.apache.samza.storage.kv.RocksDbKeyValueStorageEngineFactory",
"stores.mystore.key.serde" -> "string",
"stores.mystore.msg.serde" -> "string",
"stores.mystore.changelog" -> "kafka.mystoreChangelog",
"stores.mystore.changelog.replication.factor" -> "1",
- "systems.kafka.samza.factory" -> "org.apache.samza.system.kafka.KafkaSystemFactory",
- // Always start consuming at offset 0. This avoids a race condition between
- // the producer and the consumer in this test (SAMZA-166, SAMZA-224).
- "systems.kafka.samza.offset.default" -> "oldest", // applies to a nonempty topic
- "systems.kafka.consumer.auto.offset.reset" -> "smallest", // applies to an empty topic
- "systems.kafka.samza.msg.serde" -> "string",
- "systems.kafka.consumer.zookeeper.connect" -> zkConnect,
- "systems.kafka.producer.bootstrap.servers" -> ("localhost:%s" format port1),
- // Since using state, need a checkpoint manager
- "task.checkpoint.factory" -> "org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory",
- "task.checkpoint.system" -> "kafka",
- "task.checkpoint.replication.factor" -> "1",
// However, don't have the inputs use the checkpoint manager
// since the second part of the test expects to replay the input streams.
- "systems.kafka.streams.input.samza.reset.offset" -> "true")
+ "systems.kafka.streams.input.samza.reset.offset" -> "true"))
@Test
def testShouldStartAndRestore {
@@ -236,7 +82,7 @@ class TestStatefulTask {
// Validate that restored is empty.
assertEquals(0, task.initFinished.getCount)
- assertEquals(0, task.restored.size)
+ assertEquals(0, task.asInstanceOf[StateStoreTestTask].restored.size)
assertEquals(0, task.received.size)
// Send some messages to input stream.
@@ -248,7 +94,7 @@ class TestStatefulTask {
send(task, "-99")
// Validate that messages appear in store stream.
- val messages = readAll(STATE_TOPIC_STREAM, 5, "testShouldStartTaskForFirstTime")
+ val messages = readAll(TestStatefulTask.STATE_TOPIC_STREAM, 5, "testShouldStartTaskForFirstTime")
assertEquals(6, messages.length)
assertEquals("1", messages(0))
@@ -265,10 +111,10 @@ class TestStatefulTask {
val (job, task) = startJob
// Validate that restored has expected data.
- assertEquals(3, task.restored.size)
- assertTrue(task.restored.contains("1"))
- assertTrue(task.restored.contains("2"))
- assertTrue(task.restored.contains("3"))
+ assertEquals(3, task.asInstanceOf[StateStoreTestTask].restored.size)
+ assertTrue(task.asInstanceOf[StateStoreTestTask].restored.contains("1"))
+ assertTrue(task.asInstanceOf[StateStoreTestTask].restored.contains("2"))
+ assertTrue(task.asInstanceOf[StateStoreTestTask].restored.contains("3"))
var count = 0
@@ -290,7 +136,7 @@ class TestStatefulTask {
send(task, "5")
// Validate that messages appear in store stream.
- val messages = readAll(STATE_TOPIC_STREAM, 14, "testShouldRestoreStore")
+ val messages = readAll(TestStatefulTask.STATE_TOPIC_STREAM, 14, "testShouldRestoreStore")
assertEquals(15, messages.length)
// From initial start.
@@ -315,126 +161,25 @@ class TestStatefulTask {
stopJob(job)
}
- /**
- * Start a job for TestJob, and do some basic sanity checks around startup
- * time, number of partitions, etc.
- */
- def startJob = {
- // Start task.
- val job = new JobRunner(new MapConfig(jobConfig)).run
- assertEquals(ApplicationStatus.Running, job.waitForStatus(ApplicationStatus.Running, 60000))
- TestTask.awaitTaskRegistered
- val tasks = TestTask.tasks
- assertEquals("Should only have a single partition in this task", 1, tasks.size)
- val task = tasks.values.toList.head
- task.initFinished.await(60, TimeUnit.SECONDS)
- assertEquals(0, task.initFinished.getCount)
- (job, task)
- }
-
- /**
- * Kill a job, and wait for an unsuccessful finish (since this throws an
- * interrupt, which is forwarded on to ThreadJob, and marked as a failure).
- */
- def stopJob(job: StreamJob) {
- // Shutdown task.
- job.kill
- assertEquals(ApplicationStatus.UnsuccessfulFinish, job.waitForFinish(60000))
- }
-
- /**
- * Send a message to the input topic, and validate that it gets to the test task.
- */
- def send(task: TestTask, msg: String) {
- producer.send(new ProducerRecord(INPUT_TOPIC, msg.getBytes)).get()
- task.awaitMessage
- assertEquals(msg, task.received.last)
- }
-
- /**
- * Read all messages from a topic starting from last saved offset for group.
- * To read all from offset 0, specify a unique, new group string.
- */
- def readAll(topic: String, maxOffsetInclusive: Int, group: String): List[String] = {
- val props = new Properties
-
- props.put("zookeeper.connect", zkConnect)
- props.put("group.id", group)
- props.put("auto.offset.reset", "smallest")
-
- val consumerConfig = new ConsumerConfig(props)
- val consumerConnector = Consumer.create(consumerConfig)
- var stream = consumerConnector.createMessageStreams(Map(topic -> 1)).get(topic).get.get(0).iterator
- var message: MessageAndMetadata[Array[Byte], Array[Byte]] = null
- var messages = ArrayBuffer[String]()
-
- while (message == null || message.offset < maxOffsetInclusive) {
- message = stream.next
- if (message.message == null) {
- messages += null
- } else {
- messages += new String(message.message, "UTF-8")
- }
- System.err.println("TestStatefulTask.readAll(): offset=%s, message=%s" format (message.offset, messages.last))
- }
-
- consumerConnector.shutdown
-
- messages.toList
- }
-}
-
-object TestTask {
- val tasks = new HashMap[TaskName, TestTask] with SynchronizedMap[TaskName, TestTask]
- @volatile var allTasksRegistered = new CountDownLatch(TestStatefulTask.TOTAL_TASK_NAMES)
-
- /**
- * Static method that tasks can use to register themselves with. Useful so
- * we don't have to sneak into the ThreadJob/SamzaContainer to get our test
- * tasks.
- */
- def register(taskName: TaskName, task: TestTask) {
- tasks += taskName -> task
- allTasksRegistered.countDown
- }
-
- def awaitTaskRegistered {
- allTasksRegistered.await(60, TimeUnit.SECONDS)
- assertEquals(0, allTasksRegistered.getCount)
- assertEquals(TestStatefulTask.TOTAL_TASK_NAMES, tasks.size)
- // Reset the registered latch, so we can use it again every time we start a new job.
- TestTask.allTasksRegistered = new CountDownLatch(TestStatefulTask.TOTAL_TASK_NAMES)
- }
}
-class TestTask extends StreamTask with InitableTask {
+class StateStoreTestTask extends TestTask {
var store: KeyValueStore[String, String] = null
var restored = Set[String]()
- var received = ArrayBuffer[String]()
- val initFinished = new CountDownLatch(1)
- var gotMessage = new CountDownLatch(1)
-
- def init(config: Config, context: TaskContext) {
- TestTask.register(context.getTaskName, this)
- store = context
- .getStore(TestStatefulTask.STORE_NAME)
- .asInstanceOf[KeyValueStore[String, String]]
+
+ override def testInit(config: Config, context: TaskContext): Unit = {
+ store = context.getStore(TestStatefulTask.STORE_NAME).asInstanceOf[KeyValueStore[String, String]]
val iter = store.all
restored ++= iter
.map(_.getValue)
.toSet
- System.err.println("TestTask.init(): %s" format restored)
+ System.err.println("StateStoreTestTask.init(): %s" format restored)
iter.close
- initFinished.countDown()
}
- def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) {
+ override def testProcess(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator): Unit = {
val msg = envelope.getMessage.asInstanceOf[String]
- System.err.println("TestTask.process(): %s" format msg)
-
- received += msg
-
// A negative string means delete
if (msg.startsWith("-")) {
store.delete(msg.substring(1))
@@ -443,14 +188,6 @@ class TestTask extends StreamTask with InitableTask {
}
coordinator.commit(RequestScope.ALL_TASKS_IN_CONTAINER)
-
- // Notify sender that we got a message.
- gotMessage.countDown
- }
-
- def awaitMessage {
- assertTrue("Timed out of waiting for message rather than received one.", gotMessage.await(60, TimeUnit.SECONDS))
- assertEquals(0, gotMessage.getCount)
- gotMessage = new CountDownLatch(1)
}
}
+