You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@samza.apache.org by ja...@apache.org on 2017/09/18 17:32:51 UTC

samza git commit: SAMZA-1419; Refactor StreamProcessor tests to remove Thread.sleep()

Repository: samza
Updated Branches:
  refs/heads/master fb39a5142 -> e85b01dcb


SAMZA-1419; Refactor StreamProcessor tests to remove Thread.sleep()

Refactoring the tests to enable parallel execution. Also, removed unnecessary sleeping during execution. This PR will serve as a prototype to get some feedback on this testing pattern which eliminates state sharing between the tests and opens up the avenue for parallel execution.

This change brings down the test time for these 4 tests to **14 sec** compared to **44 secs**.

Author: Bharath Kumarasubramanian <bk...@linkedin.com>

Reviewers: Jagadish <ja...@apache.org>

Closes #291 from bharathkk/master


Project: http://git-wip-us.apache.org/repos/asf/samza/repo
Commit: http://git-wip-us.apache.org/repos/asf/samza/commit/e85b01dc
Tree: http://git-wip-us.apache.org/repos/asf/samza/tree/e85b01dc
Diff: http://git-wip-us.apache.org/repos/asf/samza/diff/e85b01dc

Branch: refs/heads/master
Commit: e85b01dcb9cfc5b774ec023b7be11998139c1aee
Parents: fb39a51
Author: Bharath Kumarasubramanian <bk...@linkedin.com>
Authored: Mon Sep 18 10:32:32 2017 -0700
Committer: Jagadish <ja...@apache.org>
Committed: Mon Sep 18 10:32:32 2017 -0700

----------------------------------------------------------------------
 .../test/processor/IdentityStreamTask.java      |  16 +-
 .../test/processor/TestStreamProcessor.java     | 149 ++++++++++++-------
 2 files changed, 99 insertions(+), 66 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/samza/blob/e85b01dc/samza-test/src/test/java/org/apache/samza/test/processor/IdentityStreamTask.java
----------------------------------------------------------------------
diff --git a/samza-test/src/test/java/org/apache/samza/test/processor/IdentityStreamTask.java b/samza-test/src/test/java/org/apache/samza/test/processor/IdentityStreamTask.java
index 783d828..4f1d1df 100644
--- a/samza-test/src/test/java/org/apache/samza/test/processor/IdentityStreamTask.java
+++ b/samza-test/src/test/java/org/apache/samza/test/processor/IdentityStreamTask.java
@@ -23,19 +23,14 @@ import org.apache.samza.config.Config;
 import org.apache.samza.system.IncomingMessageEnvelope;
 import org.apache.samza.system.OutgoingMessageEnvelope;
 import org.apache.samza.system.SystemStream;
-import org.apache.samza.task.ClosableTask;
 import org.apache.samza.task.InitableTask;
 import org.apache.samza.task.MessageCollector;
 import org.apache.samza.task.StreamTask;
 import org.apache.samza.task.TaskContext;
 import org.apache.samza.task.TaskCoordinator;
 
-import java.util.concurrent.CountDownLatch;
 
-public class IdentityStreamTask implements StreamTask , InitableTask , ClosableTask  {
-  // static field since there's no other way to share state b/w a task instance and
-  // stream processor when constructed from "task.class".
-  static CountDownLatch endLatch = new CountDownLatch(1);
+public class IdentityStreamTask implements StreamTask , InitableTask  {
   private int processedMessageCount = 0;
   private int expectedMessageCount;
   private String outputTopic;
@@ -59,14 +54,7 @@ public class IdentityStreamTask implements StreamTask , InitableTask , ClosableT
             incomingMessageEnvelope.getMessage()));
     processedMessageCount++;
     if (processedMessageCount == expectedMessageCount) {
-      endLatch.countDown();
+      taskCoordinator.shutdown(TaskCoordinator.RequestScope.ALL_TASKS_IN_CONTAINER);
     }
   }
-
-  @Override
-  public void close() throws Exception {
-    // need to create a new latch after each test since it's a static field.
-    // tests are assumed to run sequentially.
-    endLatch = new CountDownLatch(1);
-  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/samza/blob/e85b01dc/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
----------------------------------------------------------------------
diff --git a/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java b/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
index b0bfbb5..c37132f 100644
--- a/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
+++ b/samza-test/src/test/java/org/apache/samza/test/processor/TestStreamProcessor.java
@@ -21,7 +21,6 @@ package org.apache.samza.test.processor;
 
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.Iterator;
 import java.util.Map;
 import java.util.Properties;
 import java.util.concurrent.CountDownLatch;
@@ -35,6 +34,9 @@ import org.apache.kafka.clients.consumer.ConsumerRecords;
 import org.apache.kafka.clients.consumer.KafkaConsumer;
 import org.apache.kafka.clients.producer.KafkaProducer;
 import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.kafka.common.protocol.SecurityProtocol;
+import org.apache.kafka.common.serialization.ByteArraySerializer;
+import org.apache.kafka.common.serialization.StringSerializer;
 import org.apache.samza.SamzaException;
 import org.apache.samza.config.Config;
 import org.apache.samza.config.MapConfig;
@@ -47,26 +49,16 @@ import org.apache.samza.task.StreamTaskFactory;
 import org.apache.samza.test.StandaloneIntegrationTestHarness;
 import org.apache.samza.test.StandaloneTestUtils;
 import org.junit.Assert;
-import org.junit.Before;
 import org.junit.Test;
+import scala.Option$;
 
 import static org.mockito.Matchers.anyObject;
+import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doNothing;
 import static org.mockito.Mockito.mock;
-import static org.apache.samza.test.processor.IdentityStreamTask.endLatch;
 
-public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
-
-  private StreamProcessorLifecycleListener listener;
-
-  @Before
-  public void setup() {
-    listener = mock(StreamProcessorLifecycleListener.class);
-    doNothing().when(listener).onStart();
-    doNothing().when(listener).onShutdown();
-    doNothing().when(listener).onFailure(anyObject());
-  }
 
+public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
   /**
    * Testing a basic identity stream task - reads data from a topic and writes it to another topic
    * (without any modifications)
@@ -86,15 +78,11 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
     // Note: createTopics needs to be called before creating a StreamProcessor. Otherwise it fails with a
     // TopicExistsException since StreamProcessor auto-creates them.
     createTopics(inputTopic, outputTopic);
-    final StreamProcessor processor = new StreamProcessor(
-        new MapConfig(configs),
-        new HashMap<>(),
-        IdentityStreamTask::new,
-        listener);
-
-    produceMessages(inputTopic, messageCount);
-    run(processor, endLatch);
-    verifyNumMessages(outputTopic, messageCount);
+    final TestStubs stubs = new TestStubs(configs, IdentityStreamTask::new, bootstrapServers());
+
+    produceMessages(stubs.producer, inputTopic, messageCount);
+    run(stubs.processor, stubs.shutdownLatch);
+    verifyNumMessages(stubs.consumer, outputTopic, messageCount);
   }
 
   /**
@@ -109,13 +97,11 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
 
     final Config configs = new MapConfig(createConfigs("1", testSystem, inputTopic, outputTopic, messageCount));
     createTopics(inputTopic, outputTopic);
-    final StreamTaskFactory stf = IdentityStreamTask::new;
-    final StreamProcessor processor =
-        new StreamProcessor(configs, new HashMap<>(), stf, listener);
+    final TestStubs stubs = new TestStubs(configs, IdentityStreamTask::new, bootstrapServers());
 
-    produceMessages(inputTopic, messageCount);
-    run(processor, endLatch);
-    verifyNumMessages(outputTopic, messageCount);
+    produceMessages(stubs.producer, inputTopic, messageCount);
+    run(stubs.processor, stubs.shutdownLatch);
+    verifyNumMessages(stubs.consumer, outputTopic, messageCount);
   }
 
   /**
@@ -132,15 +118,11 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
     final ExecutorService executorService = Executors.newSingleThreadExecutor();
     createTopics(inputTopic, outputTopic);
     final AsyncStreamTaskFactory stf = () -> new AsyncStreamTaskAdapter(new IdentityStreamTask(), executorService);
-    final StreamProcessor processor = new StreamProcessor(
-        configs,
-        new HashMap<>(),
-        stf,
-        listener);
-
-    produceMessages(inputTopic, messageCount);
-    run(processor, endLatch);
-    verifyNumMessages(outputTopic, messageCount);
+    final TestStubs stubs = new TestStubs(configs, stf, bootstrapServers());
+
+    produceMessages(stubs.producer, inputTopic, messageCount);
+    run(stubs.processor, stubs.shutdownLatch);
+    verifyNumMessages(stubs.consumer, outputTopic, messageCount);
     executorService.shutdownNow();
   }
 
@@ -157,13 +139,9 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
     final Map<String, String> configMap = createConfigs("1", testSystem, inputTopic, outputTopic, messageCount);
     configMap.remove("task.class");
     final Config configs = new MapConfig(configMap);
+    final TestStubs stubs = new TestStubs(configs, (StreamTaskFactory) null, bootstrapServers());
 
-    StreamProcessor processor = new StreamProcessor(
-        configs,
-        new HashMap<>(),
-        (StreamTaskFactory) null,
-        listener);
-    run(processor, endLatch);
+    run(stubs.processor, stubs.shutdownLatch);
   }
 
   private void createTopics(String inputTopic, String outputTopic) {
@@ -189,8 +167,8 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
   /**
    * Produces the provided number of messages to the topic.
    */
-  private void produceMessages(String topic, int numMessages) {
-    KafkaProducer producer = getKafkaProducer();
+  @SuppressWarnings("unchecked")
+  private void produceMessages(KafkaProducer producer, String topic, int numMessages) {
     for (int i = 0; i < numMessages; i++) {
       try {
         producer.send(new ProducerRecord(topic, String.valueOf(i).getBytes())).get();
@@ -208,7 +186,6 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
     boolean latchResult = false;
     processor.start();
     try {
-      Thread.sleep(10000);
       latchResult = latch.await(10, TimeUnit.SECONDS);
     } catch (InterruptedException e) {
       e.printStackTrace();
@@ -224,8 +201,8 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
    * Consumes data from the topic until there are no new messages for a while
    * and asserts that the number of consumed messages is as expected.
    */
-  private void verifyNumMessages(String topic, int expectedNumMessages) {
-    KafkaConsumer consumer = getKafkaConsumer();
+  @SuppressWarnings("unchecked")
+  private void verifyNumMessages(KafkaConsumer consumer, String topic, int expectedNumMessages) {
     consumer.subscribe(Collections.singletonList(topic));
 
     int count = 0;
@@ -234,9 +211,7 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
     while (count < expectedNumMessages && emptyPollCount < 5) {
       ConsumerRecords records = consumer.poll(5000);
       if (!records.isEmpty()) {
-        Iterator<ConsumerRecord> iterator = records.iterator();
-        while (iterator.hasNext()) {
-          ConsumerRecord record = iterator.next();
+        for (ConsumerRecord record : (Iterable<ConsumerRecord>) records) {
           Assert.assertEquals(new String((byte[]) record.value()), String.valueOf(count));
           count++;
         }
@@ -247,4 +222,74 @@ public class TestStreamProcessor extends StandaloneIntegrationTestHarness {
 
     Assert.assertEquals(count, expectedNumMessages);
   }
+
+  /**
+   * A wrapper class to consolidate all the components required to be either mocked or stubbed prior to unit testing
+   * the stream processor.
+   */
+  private static class TestStubs {
+    CountDownLatch shutdownLatch;
+    KafkaConsumer consumer;
+    KafkaProducer producer;
+    StreamProcessor processor;
+    StreamProcessorLifecycleListener listener;
+
+    private TestStubs(String bootstrapServer) {
+      shutdownLatch = new CountDownLatch(1);
+      initProcessorListener();
+      initConsumer(bootstrapServer);
+      initProducer(bootstrapServer);
+    }
+
+    TestStubs(Config config, StreamTaskFactory taskFactory, String bootstrapServer) {
+      this(bootstrapServer);
+      processor = new StreamProcessor(config, new HashMap<>(), taskFactory, listener);
+    }
+
+    TestStubs(Config config, AsyncStreamTaskFactory taskFactory, String bootstrapServer) {
+      this(bootstrapServer);
+      processor = new StreamProcessor(config, new HashMap<>(), taskFactory, listener);
+    }
+
+    private void initConsumer(String bootstrapServer) {
+      consumer = TestUtils.createNewConsumer(
+          bootstrapServer,
+          "group",
+          "earliest",
+          4096L,
+          "org.apache.kafka.clients.consumer.RangeAssignor",
+          30000,
+          SecurityProtocol.PLAINTEXT,
+          Option$.MODULE$.empty(),
+          Option$.MODULE$.empty(),
+          Option$.MODULE$.empty());
+    }
+
+    private void initProcessorListener() {
+      listener = mock(StreamProcessorLifecycleListener.class);
+      doNothing().when(listener).onStart();
+      doNothing().when(listener).onFailure(anyObject());
+      doAnswer(invocation -> {
+          shutdownLatch.countDown();
+          return null;
+        }).when(listener).onShutdown();
+    }
+
+    private void initProducer(String bootstrapServer) {
+      producer = TestUtils.createNewProducer(
+          bootstrapServer,
+          1,
+          60 * 1000L,
+          1024L * 1024L,
+          0,
+          0L,
+          5 * 1000L,
+          SecurityProtocol.PLAINTEXT,
+          null,
+          Option$.MODULE$.apply(new Properties()),
+          new StringSerializer(),
+          new ByteArraySerializer(),
+          Option$.MODULE$.apply(new Properties()));
+    }
+  }
 }