You are viewing a plain text version of this content. The canonical link for it is here.
Posted to s4-commits@incubator.apache.org by mm...@apache.org on 2012/07/20 14:11:50 UTC
[4/4] Added checkpointing framework + tests - also simplified some
other tests - added optional checkpointing in twitter example
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/CheckpointingTest.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/CheckpointingTest.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/CheckpointingTest.java
new file mode 100644
index 0000000..ef3fabd
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/CheckpointingTest.java
@@ -0,0 +1,153 @@
+package org.apache.s4.core.ft;
+
+import java.io.File;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.Assert;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.s4.base.Event;
+import org.apache.s4.base.EventMessage;
+import org.apache.s4.base.KeyFinder;
+import org.apache.s4.comm.BareCommModule;
+import org.apache.s4.core.App;
+import org.apache.s4.core.BareCoreModule;
+import org.apache.s4.core.ProcessingElement;
+import org.apache.s4.core.Stream;
+import org.apache.s4.core.ft.FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.DummyZKStorageCallbackFactory;
+import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.NIOServerCnxn.Factory;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.Files;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+public class CheckpointingTest {
+
+ private static Factory zookeeperServerConnectionFactory = null;
+ public static File DEFAULT_TEST_OUTPUT_DIR = new File(System.getProperty("user.dir") + File.separator + "tmp");
+ public static File DEFAULT_STORAGE_DIR = new File(DEFAULT_TEST_OUTPUT_DIR.getAbsolutePath() + File.separator
+ + "storage");
+
+ @Before
+ public void prepare() throws Exception {
+ zookeeperServerConnectionFactory = CoreTestUtils.startZookeeperServer();
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ CoreTestUtils.stopZookeeperServer(zookeeperServerConnectionFactory);
+ }
+
+ @Test
+ public void testCheckpointStorage() throws Exception {
+ final ZooKeeper zk = CoreTestUtils.createZkClient();
+
+ // 2. generate a simple event that creates and changes the state of
+ // the
+ // PE
+
+ // NOTE: coordinate through zookeeper
+ final CountDownLatch signalValue1Set = new CountDownLatch(1);
+
+ CoreTestUtils.watchAndSignalCreation("/value1Set", signalValue1Set, zk);
+ final CountDownLatch signalCheckpointed = new CountDownLatch(1);
+ CoreTestUtils.watchAndSignalCreation("/checkpointed", signalCheckpointed, zk);
+
+ Injector injector = Guice.createInjector(new BareCommModule(),
+ new MockCoreModuleWithFileBaseCheckpointingBackend());
+ TestApp app = injector.getInstance(TestApp.class);
+ app.init();
+ app.start();
+
+ Event event = new Event();
+ event.put("command", String.class, "setValue1");
+ event.put("value", String.class, "message1");
+
+ app.testStream.receiveEvent(new EventMessage("", "stream1", app.getSerDeser().serialize(event)));
+
+ signalValue1Set.await();
+
+ StatefulTestPE pe = (StatefulTestPE) app.getPE("statefulPE1").getInstanceForKey("X");
+
+ Assert.assertEquals("message1", pe.getValue1());
+ Assert.assertEquals("", pe.getValue2());
+
+ // 3. generate a checkpoint event
+ event = new Event();
+ event.put("command", String.class, "checkpoint");
+ app.testStream.receiveEvent(new EventMessage("", "stream1", app.getSerDeser().serialize(event)));
+ Assert.assertTrue(signalCheckpointed.await(10, TimeUnit.SECONDS));
+
+ // NOTE: the backend has asynchronous save operations
+ Thread.sleep(1000);
+
+ CheckpointId safeKeeperId = new CheckpointId(pe);
+ File expected = new File(System.getProperty("user.dir") + File.separator + "tmp" + File.separator + "storage"
+ + File.separator + safeKeeperId.getPrototypeId() + File.separator
+ + Base64.encodeBase64URLSafeString(safeKeeperId.getStringRepresentation().getBytes()));
+
+ // 4. verify that state was correctly persisted
+ Assert.assertTrue(expected.exists());
+
+ StatefulTestPE refPE = new StatefulTestPE();
+ refPE.onCreate();
+ refPE.setValue1("message1");
+
+ Field idField = ProcessingElement.class.getDeclaredField("id");
+ idField.setAccessible(true);
+ idField.set(refPE, "X");
+
+ byte[] refBytes = app.getSerDeser().serialize(refPE);
+
+ Assert.assertTrue(Arrays.equals(refBytes, Files.toByteArray(expected)));
+
+ }
+
+ private static class TestApp extends App {
+ Stream<Event> testStream;
+ int count;
+
+ @Override
+ protected void onStart() {
+ }
+
+ @Override
+ protected void onInit() {
+
+ StatefulTestPE pe = createPE(StatefulTestPE.class, "statefulPE1");
+ testStream = createStream("stream1", new KeyFinder<Event>() {
+ @Override
+ public List<String> get(Event event) {
+ return ImmutableList.of("X");
+ }
+ }, pe);
+ }
+
+ @Override
+ protected void onClose() {
+ }
+
+ }
+
+ private static class MockCoreModuleWithFileBaseCheckpointingBackend extends BareCoreModule {
+
+ protected void configure() {
+ super.configure();
+ bind(StateStorage.class).to(DefaultFileSystemStateStorage.class);
+ bind(CheckpointingFramework.class).to(SafeKeeper.class);
+ bind(StorageCallbackFactory.class).to(DummyZKStorageCallbackFactory.class);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountApp.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountApp.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountApp.java
new file mode 100644
index 0000000..770b219
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountApp.java
@@ -0,0 +1,18 @@
+package org.apache.s4.core.ft;
+
+import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode;
+import org.apache.s4.wordcount.WordCountApp;
+
+public class FTWordCountApp extends WordCountApp {
+
+ @Override
+ protected void onInit() {
+ super.onInit();
+ getPE("classifierPE").setCheckpointingConfig(
+ new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT).frequency(1).build());
+ getPE("counterPE").setCheckpointingConfig(
+ new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT).frequency(1).build());
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountTest.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountTest.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountTest.java
new file mode 100644
index 0000000..e48d586
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FTWordCountTest.java
@@ -0,0 +1,131 @@
+package org.apache.s4.core.ft;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.Assert;
+
+import org.apache.s4.base.Event;
+import org.apache.s4.base.EventMessage;
+import org.apache.s4.base.SerializerDeserializer;
+import org.apache.s4.comm.DefaultCommModule;
+import org.apache.s4.comm.tcp.TCPEmitter;
+import org.apache.s4.core.DefaultCoreModule;
+import org.apache.s4.fixtures.CommTestUtils;
+import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.s4.fixtures.ZkBasedTest;
+import org.apache.s4.wordcount.WordCountTest;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.After;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+public class FTWordCountTest extends ZkBasedTest {
+
+ private Process forkedS4App;
+
+ @After
+ public void clean() throws IOException, InterruptedException {
+ CoreTestUtils.killS4App(forkedS4App);
+ }
+
+ @Test
+ public void testCheckpointAndRecovery() throws Exception {
+
+ Injector injector = Guice.createInjector(
+ new DefaultCommModule(Resources.getResource("default.s4.comm.properties").openStream(), "cluster1"),
+ new DefaultCoreModule(Resources.getResource("default.s4.core.properties").openStream()));
+
+ TCPEmitter emitter = injector.getInstance(TCPEmitter.class);
+
+ final ZooKeeper zk = CoreTestUtils.createZkClient();
+
+ restartNode();
+
+ CountDownLatch signalTextProcessed = new CountDownLatch(1);
+ CommTestUtils.watchAndSignalCreation("/textProcessed", signalTextProcessed, zk);
+
+ // add authorizations for processing
+ for (int i = 1; i <= WordCountTest.SENTENCE_1_TOTAL_WORDS; i++) {
+ zk.create("/continue_" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+ }
+
+ CountDownLatch signalSentence1Processed = new CountDownLatch(1);
+ CoreTestUtils.watchAndSignalCreation("/classifierIteration_" + WordCountTest.SENTENCE_1_TOTAL_WORDS,
+ signalSentence1Processed, zk);
+
+ injectSentence(injector, emitter, WordCountTest.SENTENCE_1);
+ signalSentence1Processed.await(10, TimeUnit.SECONDS);
+ Thread.sleep(1000);
+
+ // crash the app
+ forkedS4App.destroy();
+
+ restartNode();
+ // add authorizations for continuing processing. Without these, the
+ // WordClassifier processed keeps waiting
+ for (int i = WordCountTest.SENTENCE_1_TOTAL_WORDS + 1; i <= WordCountTest.SENTENCE_1_TOTAL_WORDS
+ + WordCountTest.SENTENCE_2_TOTAL_WORDS; i++) {
+ zk.create("/continue_" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+ }
+
+ CountDownLatch sentence2Processed = new CountDownLatch(1);
+ CoreTestUtils
+ .watchAndSignalCreation("/classifierIteration_"
+ + (WordCountTest.SENTENCE_1_TOTAL_WORDS + WordCountTest.SENTENCE_2_TOTAL_WORDS),
+ sentence2Processed, zk);
+
+ injectSentence(injector, emitter, WordCountTest.SENTENCE_2);
+
+ sentence2Processed.await(10, TimeUnit.SECONDS);
+ Thread.sleep(1000);
+
+ // crash the app
+ forkedS4App.destroy();
+ restartNode();
+
+ // add authorizations for continuing processing. Without these, the
+ // WordClassifier processed keeps waiting
+ for (int i = WordCountTest.SENTENCE_1_TOTAL_WORDS + WordCountTest.SENTENCE_2_TOTAL_WORDS + 1; i <= WordCountTest.TOTAL_WORDS; i++) {
+ zk.create("/continue_" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
+ }
+ injectSentence(injector, emitter, WordCountTest.SENTENCE_3);
+ signalTextProcessed.await(10, TimeUnit.SECONDS);
+ File results = new File(CommTestUtils.DEFAULT_TEST_OUTPUT_DIR + File.separator + "wordcount");
+ if (!results.exists()) {
+ // in case the results file isn't ready yet
+ Thread.sleep(1000);
+ results = new File(CommTestUtils.DEFAULT_TEST_OUTPUT_DIR + File.separator + "wordcount");
+ }
+ String s = CoreTestUtils.readFile(results);
+ Assert.assertEquals("be=2;da=2;doobie=5;not=1;or=1;to=2;", s);
+
+ }
+
+ private void injectSentence(Injector injector, TCPEmitter emitter, String sentence) {
+ Event event;
+ event = new Event();
+ event.put("sentence", String.class, sentence);
+ emitter.send(0, new EventMessage("-1", "inputStream", injector.getInstance(SerializerDeserializer.class)
+ .serialize(event)));
+ }
+
+ private void restartNode() throws IOException, InterruptedException {
+ CountDownLatch signalConsumerReady = RecoveryTest.getConsumerReadySignal("inputStream");
+
+ // recovering and making sure checkpointing still works
+ forkedS4App = CoreTestUtils.forkS4Node(new String[] { "-c", "cluster1", "-appClass",
+ FTWordCountApp.class.getName(), "-p",
+ "{s4.checkpointing.filesystem.storageRootPath=" + CommTestUtils.DEFAULT_STORAGE_DIR + "}",
+ "-extraModulesClasses", FileSystemBackendCheckpointingModule.class.getName() });
+ Assert.assertTrue(signalConsumerReady.await(20, TimeUnit.SECONDS));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.java
new file mode 100644
index 0000000..3787ef7
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.java
@@ -0,0 +1,50 @@
+package org.apache.s4.core.ft;
+
+import org.apache.s4.core.ft.CheckpointingFramework.StorageResultCode;
+import org.apache.s4.fixtures.CommTestUtils;
+import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+import com.google.inject.name.Names;
+
+/**
+ * Creates the /checkpointed znode if a successful checkpointing callback is received. Does it only once.
+ *
+ */
+public class FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule extends
+ FileSystemBackendCheckpointingModule {
+
+ @Override
+ protected void configure() {
+ super.configure();
+ bind(StorageCallbackFactory.class).to(DummyZKStorageCallbackFactory.class);
+ bind(String.class).annotatedWith(Names.named("s4.checkpointing.filesystem.storageRootPath")).toInstance(
+ CommTestUtils.DEFAULT_STORAGE_DIR.getAbsolutePath());
+ }
+
+ public static class DummyZKStorageCallbackFactory implements StorageCallbackFactory {
+
+ @Override
+ public StorageCallback createStorageCallback() {
+ return new StorageCallback() {
+
+ @Override
+ public void storageOperationResult(StorageResultCode resultCode, Object message) {
+ if (resultCode == StorageResultCode.SUCCESS) {
+ try {
+ ZooKeeper zkClient = CoreTestUtils.createZkClient();
+ zkClient.create("/checkpointed", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ }
+ };
+ };
+
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/RecoveryTest.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/RecoveryTest.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/RecoveryTest.java
new file mode 100644
index 0000000..326e1a1
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/RecoveryTest.java
@@ -0,0 +1,158 @@
+package org.apache.s4.core.ft;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import junit.framework.Assert;
+
+import org.I0Itec.zkclient.IZkChildListener;
+import org.apache.s4.base.Event;
+import org.apache.s4.base.EventMessage;
+import org.apache.s4.base.SerializerDeserializer;
+import org.apache.s4.comm.DefaultCommModule;
+import org.apache.s4.comm.tcp.TCPEmitter;
+import org.apache.s4.comm.topology.ZkClient;
+import org.apache.s4.core.DefaultCoreModule;
+import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.s4.fixtures.ZkBasedTest;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.After;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+public class RecoveryTest extends ZkBasedTest {
+
+ private Process forkedS4App = null;
+
+ @After
+ public void cleanup() throws Exception {
+ CoreTestUtils.killS4App(forkedS4App);
+ }
+
+ @Test
+ public void testCheckpointRestorationThroughApplicationEvent() throws Exception {
+
+ testCheckpointingConfiguration(S4AppWithManualCheckpointing.class,
+ FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.class, true,
+ "value1=message1 ; value2=message2");
+
+ }
+
+ @Test
+ public void testEventCountBasedCheckpointingAndRecovery() throws Exception {
+
+ testCheckpointingConfiguration(S4AppWithCountBasedCheckpointing.class,
+ FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.class, false,
+ "value1=message1 ; value2=message2");
+
+ }
+
+ @Test
+ public void testTimeBasedCheckpointingAndRecovery() throws Exception {
+ testCheckpointingConfiguration(S4AppWithTimeBasedCheckpointing.class,
+ FileSystemBasedBackendWithZKStorageCallbackCheckpointingModule.class, false,
+ "value1=message1 ; value2=message2");
+ }
+
+ @Test
+ public void testTimingOutRecovery() throws Exception {
+ testCheckpointingConfiguration(S4AppWithCountBasedCheckpointing.class,
+ CheckpointingModuleWithUnrespondingFetchingStorageBackend.class, false, "value1= ; value2=message2");
+ }
+
+ private void insertCheckpointInstruction(Injector injector, TCPEmitter emitter) {
+ Event event;
+ event = new Event();
+ event.put("command", String.class, "checkpoint");
+ emitter.send(0, new EventMessage("-1", "inputStream", injector.getInstance(SerializerDeserializer.class)
+ .serialize(event)));
+ }
+
+ private void testCheckpointingConfiguration(Class<?> appClass, Class<?> backendModuleClass,
+ boolean manualCheckpointing, String expectedFinalResult) throws IOException, InterruptedException,
+ KeeperException {
+ // here checkpointing is automatic for every event: no need to send a "checkpoint" event. The checkpointing
+ // configuration is specified in the app (S4AppWithCountBasedCheckpointing class)
+
+ final ZooKeeper zk = CoreTestUtils.createZkClient();
+
+ // use a latch for waiting for app to be ready
+ CountDownLatch signalConsumerReady = getConsumerReadySignal("inputStream");
+
+ // 1. instantiate remote S4 app
+ forkedS4App = CoreTestUtils.forkS4Node(new String[] { "-c", "cluster1", "-appClass", appClass.getName(),
+ "-extraModulesClasses", backendModuleClass.getName() });
+
+ Assert.assertTrue(signalConsumerReady.await(20, TimeUnit.SECONDS));
+
+ CountDownLatch signalValue1Set = new CountDownLatch(1);
+ CoreTestUtils.watchAndSignalCreation("/value1Set", signalValue1Set, zk);
+ final CountDownLatch signalCheckpointed = new CountDownLatch(1);
+ CoreTestUtils.watchAndSignalCreation("/checkpointed", signalCheckpointed, zk);
+
+ Injector injector = Guice.createInjector(
+ new DefaultCommModule(Resources.getResource("default.s4.comm.properties").openStream(), "cluster1"),
+ new DefaultCoreModule(Resources.getResource("default.s4.core.properties").openStream()));
+
+ TCPEmitter emitter = injector.getInstance(TCPEmitter.class);
+
+ Event event = new Event();
+ event.put("command", String.class, "setValue1");
+ event.put("value", String.class, "message1");
+ emitter.send(0, new EventMessage("-1", "inputStream", injector.getInstance(SerializerDeserializer.class)
+ .serialize(event)));
+
+ if (manualCheckpointing) {
+ insertCheckpointInstruction(injector, emitter);
+ }
+
+ Assert.assertTrue(signalCheckpointed.await(10, TimeUnit.SECONDS));
+
+ forkedS4App.destroy();
+
+ zk.delete("/data", -1);
+
+ signalConsumerReady = getConsumerReadySignal("inputStream");
+ forkedS4App = CoreTestUtils.forkS4Node(new String[] { "-c", "cluster1", "-appClass",
+ S4AppWithManualCheckpointing.class.getName(), "-extraModulesClasses", backendModuleClass.getName() });
+
+ Assert.assertTrue(signalConsumerReady.await(20, TimeUnit.SECONDS));
+ // // trigger recovery by sending application event to set value 2
+ CountDownLatch signalValue2Set = new CountDownLatch(1);
+ CoreTestUtils.watchAndSignalCreation("/value2Set", signalValue2Set, zk);
+
+ event = new Event();
+ event.put("command", String.class, "setValue2");
+ event.put("value", String.class, "message2");
+ emitter.send(0, new EventMessage("-1", "inputStream", injector.getInstance(SerializerDeserializer.class)
+ .serialize(event)));
+
+ Assert.assertTrue(signalValue2Set.await(10, TimeUnit.SECONDS));
+
+ Assert.assertEquals(expectedFinalResult, new String(zk.getData("/data", false, null)));
+ }
+
+ public static CountDownLatch getConsumerReadySignal(String streamName) {
+ final CountDownLatch signalAppReady = new CountDownLatch(1);
+
+ ZkClient zkClient = new ZkClient("localhost:" + CoreTestUtils.ZK_PORT);
+ // TODO check a proper app state variable. This is hacky
+ zkClient.subscribeChildChanges("/s4/streams/" + streamName + "/consumers", new IZkChildListener() {
+
+ @Override
+ public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
+ if (currentChilds.size() == 1) {
+ signalAppReady.countDown();
+ }
+
+ }
+ });
+ return signalAppReady;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithCountBasedCheckpointing.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithCountBasedCheckpointing.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithCountBasedCheckpointing.java
new file mode 100644
index 0000000..0ad4c0d
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithCountBasedCheckpointing.java
@@ -0,0 +1,24 @@
+package org.apache.s4.core.ft;
+
+import org.apache.s4.core.App;
+import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode;
+
+public class S4AppWithCountBasedCheckpointing extends App {
+
+ @Override
+ protected void onStart() {
+ }
+
+ @Override
+ protected void onInit() {
+ StatefulTestPE pe = createPE(StatefulTestPE.class);
+ pe.setSingleton(true);
+ pe.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT).frequency(1).build());
+ createInputStream("inputStream", pe);
+ }
+
+ @Override
+ protected void onClose() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithManualCheckpointing.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithManualCheckpointing.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithManualCheckpointing.java
new file mode 100644
index 0000000..2971c57
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithManualCheckpointing.java
@@ -0,0 +1,29 @@
+package org.apache.s4.core.ft;
+
+import org.apache.s4.core.App;
+
+/**
+ *
+ *
+ */
+public class S4AppWithManualCheckpointing extends App {
+
+ @Override
+ protected void onStart() {
+
+ }
+
+ @Override
+ protected void onInit() {
+ StatefulTestPE pe = createPE(StatefulTestPE.class);
+ pe.setSingleton(true);
+ createInputStream("inputStream", pe);
+ }
+
+ @Override
+ protected void onClose() {
+ // TODO Auto-generated method stub
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithTimeBasedCheckpointing.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithTimeBasedCheckpointing.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithTimeBasedCheckpointing.java
new file mode 100644
index 0000000..910f882
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/S4AppWithTimeBasedCheckpointing.java
@@ -0,0 +1,28 @@
+package org.apache.s4.core.ft;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.s4.core.App;
+import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode;
+
+public class S4AppWithTimeBasedCheckpointing extends App {
+
+ @Override
+ protected void onStart() {
+ }
+
+ @Override
+ protected void onInit() {
+ StatefulTestPE pe = createPE(StatefulTestPE.class);
+ pe.setSingleton(true);
+ // checkpoints (if applicable) every 1 ms!
+ pe.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.TIME).frequency(1)
+ .timeUnit(TimeUnit.MILLISECONDS).build());
+ createInputStream("inputStream", pe);
+ }
+
+ @Override
+ protected void onClose() {
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StatefulTestPE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StatefulTestPE.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StatefulTestPE.java
new file mode 100644
index 0000000..70e321d
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StatefulTestPE.java
@@ -0,0 +1,97 @@
+package org.apache.s4.core.ft;
+
+import java.io.IOException;
+
+import org.apache.s4.core.ProcessingElement;
+import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.KeeperException.NoNodeException;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+
+public class StatefulTestPE extends ProcessingElement {
+
+ String id;
+ String value1 = "";
+ String value2 = "";
+ transient ZooKeeper zk = null;
+
+ public void onEvent(org.apache.s4.base.Event event) {
+ try {
+
+ if ("setValue1".equals(event.get("command"))) {
+ setValue1(event.get("value"));
+ zk.create("/value1Set", event.get("value").getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } else if ("setValue2".equals(event.get("command"))) {
+ setValue2(event.get("value"));
+ zk.create("/value2Set", event.get("value").getBytes(), Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
+ } else if ("checkpoint".equals(event.get("command"))) {
+ checkpoint();
+ } else {
+ throw new RuntimeException("unidentified event: " + event);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ public String getValue1() {
+ return value1;
+ }
+
+ public void setValue1(String value1) {
+ this.value1 = value1;
+ persistValues();
+ }
+
+ public String getValue2() {
+ return value2;
+ }
+
+ public void setValue2(String value2) {
+ this.value2 = value2;
+ persistValues();
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ // NOTE: we use a file as a simple way to keep track of changes
+ private void persistValues() {
+
+ try {
+ try {
+ zk.delete("/data", -1);
+ } catch (NoNodeException ignored) {
+ }
+ zk.create("/data", ("value1=" + value1 + " ; value2=" + value2).getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } catch (KeeperException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ protected void onCreate() {
+ try {
+ zk = CoreTestUtils.createZkClient();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ @Override
+ protected void onRemove() {
+ // TODO Auto-generated method stub
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StorageWithUnrespondingFetching.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StorageWithUnrespondingFetching.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StorageWithUnrespondingFetching.java
new file mode 100644
index 0000000..2906fe8
--- /dev/null
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/ft/StorageWithUnrespondingFetching.java
@@ -0,0 +1,45 @@
+package org.apache.s4.core.ft;
+
+import java.util.Set;
+
+import com.google.inject.Inject;
+import com.google.inject.name.Named;
+
+public class StorageWithUnrespondingFetching implements StateStorage {
+
+ @Inject
+ DefaultFileSystemStateStorage storage = new DefaultFileSystemStateStorage();
+
+ @Inject
+ @Named("s4.checkpointing.filesystem.storageRootPath")
+ String storageRootPath;
+
+ @Inject
+ private void initStorageRootPath() {
+ // manual init because we are not directly injecting the default file system storage, but creating a new
+ // instance in this class
+ storage.storageRootPath = this.storageRootPath;
+ }
+
+ @Override
+ public void saveState(CheckpointId key, byte[] state, StorageCallback callback) {
+ storage.saveState(key, state, callback);
+ }
+
+ @Override
+ public byte[] fetchState(CheckpointId key) {
+ try {
+ Thread.sleep(10000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ Thread.currentThread().interrupt();
+ }
+ return storage.fetchState(key);
+ }
+
+ @Override
+ public Set<CheckpointId> fetchStoredKeys() {
+ return storage.fetchStoredKeys();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggerablePE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggerablePE.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggerablePE.java
index e72073c..6f1cc9f 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggerablePE.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggerablePE.java
@@ -31,8 +31,7 @@ public class TriggerablePE extends ProcessingElement implements Watcher {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ Thread.currentThread().interrupt();
}
}
@@ -55,8 +54,7 @@ public class TriggerablePE extends ProcessingElement implements Watcher {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
+ Thread.currentThread().interrupt();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggeredApp.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggeredApp.java b/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggeredApp.java
index 849d7b2..e2a1b38 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggeredApp.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/core/triggers/TriggeredApp.java
@@ -1,21 +1,18 @@
package org.apache.s4.core.triggers;
-import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.s4.base.Event;
import org.apache.s4.core.App;
import org.apache.s4.core.Stream;
import org.apache.s4.core.TriggerTest;
-import org.apache.s4.fixtures.SocketAdapter;
import org.apache.s4.wordcount.SentenceKeyFinder;
-import org.apache.s4.wordcount.StringEvent;
import com.google.inject.Inject;
public class TriggeredApp extends App {
- SocketAdapter<StringEvent> socketAdapter;
+ public Stream<Event> stream;
@Inject
public TriggeredApp() {
@@ -32,7 +29,7 @@ public class TriggeredApp extends App {
protected void onInit() {
TriggerablePE prototype = createPE(TriggerablePE.class);
- Stream<StringEvent> stream = createStream("stream", new SentenceKeyFinder(), prototype);
+ stream = createStream("stream", new SentenceKeyFinder(), prototype);
switch (TriggerTest.triggerType) {
case COUNT_BASED:
prototype.setTrigger(Event.class, 1, 0, TimeUnit.SECONDS);
@@ -43,11 +40,6 @@ public class TriggeredApp extends App {
break;
}
- try {
- socketAdapter = new SocketAdapter<StringEvent>(stream, new SocketAdapter.SentenceEventFactory());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/deploy/TestAutomaticDeployment.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/deploy/TestAutomaticDeployment.java b/subprojects/s4-core/src/test/java/org/apache/s4/deploy/TestAutomaticDeployment.java
index 108a144..620494f 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/deploy/TestAutomaticDeployment.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/deploy/TestAutomaticDeployment.java
@@ -15,23 +15,30 @@ import junit.framework.Assert;
import org.I0Itec.zkclient.IZkChildListener;
import org.I0Itec.zkclient.ZkClient;
import org.apache.commons.configuration.ConfigurationException;
-import org.apache.s4.comm.tools.TaskSetup;
+import org.apache.s4.base.Event;
+import org.apache.s4.base.EventMessage;
+import org.apache.s4.base.SerializerDeserializer;
+import org.apache.s4.comm.DefaultCommModule;
+import org.apache.s4.comm.tcp.TCPEmitter;
import org.apache.s4.comm.topology.ZNRecord;
import org.apache.s4.comm.topology.ZNRecordSerializer;
+import org.apache.s4.core.DefaultCoreModule;
import org.apache.s4.fixtures.CommTestUtils;
import org.apache.s4.fixtures.CoreTestUtils;
+import org.apache.s4.fixtures.ZkBasedTest;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.server.NIOServerCnxn.Factory;
import org.jboss.netty.handler.codec.http.HttpHeaders;
import org.junit.After;
-import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.io.ByteStreams;
import com.google.common.io.Files;
+import com.google.common.io.Resources;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
import com.sun.net.httpserver.Headers;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
@@ -45,12 +52,11 @@ import com.sun.net.httpserver.HttpServer;
* - ... or from a web server
*
*/
-public class TestAutomaticDeployment {
+public class TestAutomaticDeployment extends ZkBasedTest {
private Factory zookeeperServerConnectionFactory;
private Process forkedNode;
private ZkClient zkClient;
- private static final String CLUSTER_NAME = "clusterZ";
private HttpServer httpServer;
private static File tmpAppsDir;
@@ -95,7 +101,7 @@ public class TestAutomaticDeployment {
ZNRecord record = new ZNRecord(String.valueOf(System.currentTimeMillis()));
record.putSimpleField(DistributedDeploymentManager.S4R_URI, uri);
- zkClient.create("/s4/clusters/" + CLUSTER_NAME + "/app/s4App", record, CreateMode.PERSISTENT);
+ zkClient.create("/s4/clusters/cluster1/app/s4App", record, CreateMode.PERSISTENT);
Assert.assertTrue(signalAppInitialized.await(20, TimeUnit.SECONDS));
Assert.assertTrue(signalAppStarted.await(20, TimeUnit.SECONDS));
@@ -106,7 +112,16 @@ public class TestAutomaticDeployment {
CommTestUtils
.watchAndSignalCreation("/onEvent@" + time1, signalEvent1Processed, CommTestUtils.createZkClient());
- CoreTestUtils.injectIntoStringSocketAdapter(time1);
+ Injector injector = Guice.createInjector(
+ new DefaultCommModule(Resources.getResource("default.s4.comm.properties").openStream(), "cluster1"),
+ new DefaultCoreModule(Resources.getResource("default.s4.core.properties").openStream()));
+
+ TCPEmitter emitter = injector.getInstance(TCPEmitter.class);
+
+ Event event = new Event();
+ event.put("line", String.class, time1);
+ emitter.send(0, new EventMessage("-1", "inputStream", injector.getInstance(SerializerDeserializer.class)
+ .serialize(event)));
// check event processed
Assert.assertTrue(signalEvent1Processed.await(5, TimeUnit.SECONDS));
@@ -148,17 +163,13 @@ public class TestAutomaticDeployment {
// current package .
// 1. start s4 nodes. Check that no app is deployed.
- TaskSetup taskSetup = new TaskSetup("localhost:" + CommTestUtils.ZK_PORT);
- taskSetup.clean(CLUSTER_NAME);
- taskSetup.setup(CLUSTER_NAME, 1, 1300);
-
zkClient = new ZkClient("localhost:" + CommTestUtils.ZK_PORT);
zkClient.setZkSerializer(new ZNRecordSerializer());
- List<String> processes = zkClient.getChildren("/s4/clusters/" + CLUSTER_NAME + "/process");
+ List<String> processes = zkClient.getChildren("/s4/clusters/cluster1/process");
Assert.assertTrue(processes.size() == 0);
final CountDownLatch signalProcessesReady = new CountDownLatch(1);
- zkClient.subscribeChildChanges("/s4/clusters/" + CLUSTER_NAME + "/process", new IZkChildListener() {
+ zkClient.subscribeChildChanges("/s4/clusters/cluster1/process", new IZkChildListener() {
@Override
public void handleChildChange(String parentPath, List<String> currentChilds) throws Exception {
@@ -169,7 +180,7 @@ public class TestAutomaticDeployment {
}
});
- forkedNode = CoreTestUtils.forkS4Node(new String[] { "-cluster=" + CLUSTER_NAME });
+ forkedNode = CoreTestUtils.forkS4Node(new String[] { "-cluster=cluster1" });
// TODO synchro with ready state from zk
Thread.sleep(10000);
@@ -177,22 +188,19 @@ public class TestAutomaticDeployment {
}
- @Before
- public void prepare() throws Exception {
- CommTestUtils.cleanupTmpDirs();
- zookeeperServerConnectionFactory = CommTestUtils.startZookeeperServer();
- final ZooKeeper zk = CommTestUtils.createZkClient();
- try {
- zk.delete("/simpleAppCreated", -1);
- } catch (Exception ignored) {
- }
-
- zk.close();
- }
+ // @Before
+ // public void clean() throws Exception {
+ // final ZooKeeper zk = CommTestUtils.createZkClient();
+ // try {
+ // zk.delete("/simpleAppCreated", -1);
+ // } catch (Exception ignored) {
+ // }
+ //
+ // zk.close();
+ // }
@After
public void cleanup() throws Exception {
- CommTestUtils.stopZookeeperServer(zookeeperServerConnectionFactory);
CommTestUtils.killS4App(forkedNode);
if (httpServer != null) {
httpServer.stop(0);
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/CoreTestUtils.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/CoreTestUtils.java b/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/CoreTestUtils.java
index 832ce1b..00a7611 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/CoreTestUtils.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/CoreTestUtils.java
@@ -7,7 +7,6 @@ import java.util.List;
import junit.framework.Assert;
-import org.apache.s4.core.App;
import org.apache.s4.core.Main;
import org.gradle.tooling.BuildLauncher;
import org.gradle.tooling.GradleConnector;
@@ -25,10 +24,6 @@ import com.google.common.io.PatternFilenameFilter;
*/
public class CoreTestUtils extends CommTestUtils {
- public static Process forkS4App(Class<?> moduleClass, Class<?> appClass) throws IOException, InterruptedException {
- return forkProcess(App.class.getName(), -1, moduleClass.getName(), appClass.getName());
- }
-
public static Process forkS4Node() throws IOException, InterruptedException {
return forkS4Node(new String[] {});
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/SocketAdapter.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/SocketAdapter.java b/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/SocketAdapter.java
deleted file mode 100644
index c1815bf..0000000
--- a/subprojects/s4-core/src/test/java/org/apache/s4/fixtures/SocketAdapter.java
+++ /dev/null
@@ -1,101 +0,0 @@
-package org.apache.s4.fixtures;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.ServerSocket;
-import java.net.Socket;
-
-import org.apache.s4.core.Stream;
-import org.apache.s4.wordcount.KeyValueEvent;
-import org.apache.s4.wordcount.StringEvent;
-
-public class SocketAdapter<T extends StringEvent> {
-
- static ServerSocket serverSocket;
-
- /**
- * Listens to incoming sentence and forwards them to a sentence Stream. Each sentence is sent through a new socket
- * connection
- *
- * @param stream
- * @throws IOException
- */
- public SocketAdapter(final Stream<T> stream, final StringEventFactory<T> stringEventFactory) throws IOException {
- Thread t = new Thread(new Runnable() {
-
- @Override
- public void run() {
- serverSocket = null;
- Socket connectedSocket;
- BufferedReader in = null;
- try {
- serverSocket = new ServerSocket(12000);
- while (true) {
- connectedSocket = serverSocket.accept();
- in = new BufferedReader(new InputStreamReader(connectedSocket.getInputStream()));
-
- String line = in.readLine();
- System.out.println("read: " + line);
- stream.put(stringEventFactory.create(line));
- connectedSocket.close();
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- // System.exit(-1);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- if (serverSocket != null) {
- try {
- serverSocket.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
- });
- t.start();
-
- }
-
- public void close() {
- if (serverSocket != null) {
- try {
- serverSocket.close();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- interface StringEventFactory<T> {
- T create(String string);
- }
-
- public static class SentenceEventFactory implements StringEventFactory<StringEvent> {
-
- @Override
- public StringEvent create(String string) {
- return new StringEvent(string);
- }
-
- }
-
- public static class KeyValueEventFactory implements StringEventFactory<KeyValueEvent> {
-
- @Override
- public KeyValueEvent create(String string) {
- return new KeyValueEvent(string);
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/SentenceKeyFinder.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/SentenceKeyFinder.java b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/SentenceKeyFinder.java
index 2401f70..b234ff3 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/SentenceKeyFinder.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/SentenceKeyFinder.java
@@ -1,19 +1,19 @@
package org.apache.s4.wordcount;
-
-import java.util.ArrayList;
import java.util.List;
+import org.apache.s4.base.Event;
import org.apache.s4.base.KeyFinder;
-public class SentenceKeyFinder implements KeyFinder<StringEvent> {
+import com.google.common.collect.ImmutableList;
+
+public class SentenceKeyFinder implements KeyFinder<Event> {
private static final String SENTENCE_KEY = "sentence";
- @SuppressWarnings("serial")
@Override
- public List<String> get(StringEvent event) {
- return new ArrayList<String>(){{add(SENTENCE_KEY);}};
+ public List<String> get(Event event) {
+ return ImmutableList.of(SENTENCE_KEY);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountApp.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountApp.java b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountApp.java
index 278b752..cda0ab4 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountApp.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountApp.java
@@ -1,17 +1,14 @@
package org.apache.s4.wordcount;
-import java.io.IOException;
-
+import org.apache.s4.base.Event;
import org.apache.s4.core.App;
import org.apache.s4.core.Stream;
-import org.apache.s4.fixtures.SocketAdapter;
import com.google.inject.Inject;
public class WordCountApp extends App {
protected boolean checkpointing = false;
- SocketAdapter<StringEvent> socketAdapter;
@Inject
public WordCountApp() {
@@ -27,42 +24,21 @@ public class WordCountApp extends App {
@Override
protected void onInit() {
- WordClassifierPE wordClassifierPrototype = createPE(WordClassifierPE.class);
+ WordClassifierPE wordClassifierPrototype = createPE(WordClassifierPE.class, "classifierPE");
Stream<WordCountEvent> wordCountStream = createStream("words counts stream", new WordCountKeyFinder(),
wordClassifierPrototype);
- WordCounterPE wordCounterPrototype = createPE(WordCounterPE.class);
+ WordCounterPE wordCounterPrototype = createPE(WordCounterPE.class, "counterPE");
// wordCounterPrototype.setTrigger(WordSeenEvent.class, 1, 0, null);
wordCounterPrototype.setWordClassifierStream(wordCountStream);
Stream<WordSeenEvent> wordSeenStream = createStream("words seen stream", new WordSeenKeyFinder(),
wordCounterPrototype);
WordSplitterPE wordSplitterPrototype = createPE(WordSplitterPE.class);
wordSplitterPrototype.setWordSeenStream(wordSeenStream);
- Stream<StringEvent> sentenceStream = createStream("sentences stream", new SentenceKeyFinder(),
- wordSplitterPrototype);
-
- try {
- socketAdapter = new SocketAdapter<StringEvent>(sentenceStream, new SocketAdapter.SentenceEventFactory());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- if (checkpointing) {
-
- // TODO move to subclass
-
- // checkpoint word classifier because it maintains a sync counter
- // for the test
- // LoggerFactory.getLogger(getClass()).info("setting checkpointing for word classifier and word counter");
- // wordClassifierPrototype.setCheckpointingFrequency(1);
- // // checkpoint word counter because it maintains word counts
- // wordCounterPrototype.setCheckpointingFrequency(1);
- }
-
+ Stream<Event> sentenceStream = createInputStream("inputStream", new SentenceKeyFinder(), wordSplitterPrototype);
}
@Override
protected void onClose() {
- socketAdapter.close();
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java
index 2365a48..5987e4a 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordCountTest.java
@@ -3,22 +3,30 @@ package org.apache.s4.wordcount;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import junit.framework.Assert;
-import org.apache.s4.comm.tools.TaskSetup;
+import org.apache.s4.base.Event;
+import org.apache.s4.base.EventMessage;
+import org.apache.s4.base.SerializerDeserializer;
+import org.apache.s4.comm.DefaultCommModule;
+import org.apache.s4.comm.tcp.TCPEmitter;
+import org.apache.s4.core.DefaultCoreModule;
import org.apache.s4.core.Main;
import org.apache.s4.fixtures.CommTestUtils;
+import org.apache.s4.fixtures.ZkBasedTest;
import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.ZooDefs.Ids;
import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.server.NIOServerCnxn.Factory;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-public class WordCountTest {
+import com.google.common.io.Resources;
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+
+public class WordCountTest extends ZkBasedTest {
public static final String SENTENCE_1 = "to be or not to be doobie doobie da";
public static final int SENTENCE_1_TOTAL_WORDS = SENTENCE_1.split(" ").length;
@@ -28,12 +36,25 @@ public class WordCountTest {
public static final int SENTENCE_3_TOTAL_WORDS = SENTENCE_3.split(" ").length;
public static final String FLAG = ";";
public static int TOTAL_WORDS = SENTENCE_1_TOTAL_WORDS + SENTENCE_2_TOTAL_WORDS + SENTENCE_3_TOTAL_WORDS;
- private static Factory zookeeperServerConnectionFactory;
+ private TCPEmitter emitter;
+ private Injector injector;
+
+ // private static Factory zookeeperServerConnectionFactory;
+
+ // @Before
+ // public void prepare() throws IOException, InterruptedException, KeeperException {
+ // CommTestUtils.cleanupTmpDirs();
+ // zookeeperServerConnectionFactory = CommTestUtils.startZookeeperServer();
+ //
+ // }
@Before
- public void prepare() throws IOException, InterruptedException, KeeperException {
- CommTestUtils.cleanupTmpDirs();
- zookeeperServerConnectionFactory = CommTestUtils.startZookeeperServer();
+ public void prepareEmitter() throws IOException {
+ injector = Guice.createInjector(new DefaultCommModule(Resources.getResource("default.s4.comm.properties")
+ .openStream(), "cluster1"), new DefaultCoreModule(Resources.getResource("default.s4.core.properties")
+ .openStream()));
+
+ emitter = injector.getInstance(TCPEmitter.class);
}
@@ -54,12 +75,8 @@ public class WordCountTest {
@Test
public void testSimple() throws Exception {
final ZooKeeper zk = CommTestUtils.createZkClient();
- TaskSetup taskSetup = new TaskSetup("localhost:" + CommTestUtils.ZK_PORT);
- String clusterName = "clusterA";
- taskSetup.clean("s4");
- taskSetup.setup(clusterName, 1, 10000);
- Main.main(new String[] { "-cluster=" + clusterName, "-appClass=" + WordCountApp.class.getName(),
+ Main.main(new String[] { "-cluster=cluster1", "-appClass=" + WordCountApp.class.getName(),
"-extraModulesClasses=" + WordCountModule.class.getName() });
CountDownLatch signalTextProcessed = new CountDownLatch(1);
@@ -69,20 +86,21 @@ public class WordCountTest {
for (int i = 1; i <= SENTENCE_1_TOTAL_WORDS + SENTENCE_2_TOTAL_WORDS + 1; i++) {
zk.create("/continue_" + i, new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL);
}
- CommTestUtils.injectIntoStringSocketAdapter(SENTENCE_1);
- CommTestUtils.injectIntoStringSocketAdapter(SENTENCE_2);
- CommTestUtils.injectIntoStringSocketAdapter(SENTENCE_3);
- signalTextProcessed.await();
+ injectSentence(SENTENCE_1);
+ injectSentence(SENTENCE_2);
+ injectSentence(SENTENCE_3);
+ Assert.assertTrue(signalTextProcessed.await(10, TimeUnit.SECONDS));
File results = new File(CommTestUtils.DEFAULT_TEST_OUTPUT_DIR + File.separator + "wordcount");
String s = CommTestUtils.readFile(results);
Assert.assertEquals("be=2;da=2;doobie=5;not=1;or=1;to=2;", s);
}
- @After
- public void cleanup() throws IOException, InterruptedException {
- CommTestUtils.stopZookeeperServer(zookeeperServerConnectionFactory);
-
+ public void injectSentence(String sentence) throws IOException {
+ Event event = new Event();
+ event.put("sentence", String.class, sentence);
+ emitter.send(0, new EventMessage("-1", "inputStream", injector.getInstance(SerializerDeserializer.class)
+ .serialize(event)));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSplitterPE.java
----------------------------------------------------------------------
diff --git a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSplitterPE.java b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSplitterPE.java
index 745f521..b2a43a9 100644
--- a/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSplitterPE.java
+++ b/subprojects/s4-core/src/test/java/org/apache/s4/wordcount/WordSplitterPE.java
@@ -1,26 +1,25 @@
package org.apache.s4.wordcount;
+import org.apache.s4.base.Event;
import org.apache.s4.core.App;
import org.apache.s4.core.ProcessingElement;
import org.apache.s4.core.Stream;
-
public class WordSplitterPE extends ProcessingElement {
-
+
private Stream<WordSeenEvent> wordSeenStream;
public WordSplitterPE(App app) {
super(app);
}
- public void onEvent(StringEvent event) {
- StringEvent sentence = event;
- String[] split = sentence.getString().split(" ");
+ public void onEvent(Event event) {
+ String[] split = event.get("sentence").split(" ");
for (String word : split) {
wordSeenStream.put(new WordSeenEvent(word));
}
}
-
+
public void setWordSeenStream(Stream<WordSeenEvent> stream) {
this.wordSeenStream = stream;
}
@@ -28,13 +27,13 @@ public class WordSplitterPE extends ProcessingElement {
@Override
protected void onCreate() {
// TODO Auto-generated method stub
-
+
}
@Override
protected void onRemove() {
// TODO Auto-generated method stub
-
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/SocketAdapter.java
----------------------------------------------------------------------
diff --git a/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/SocketAdapter.java b/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/SocketAdapter.java
deleted file mode 100644
index 5ee7197..0000000
--- a/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/SocketAdapter.java
+++ /dev/null
@@ -1,80 +0,0 @@
-package org.apache.s4.deploy;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.ServerSocket;
-import java.net.Socket;
-
-import org.apache.s4.base.Event;
-import org.apache.s4.core.Stream;
-
-public class SocketAdapter {
-
- static ServerSocket serverSocket;
-
- /**
- * Listens to incoming sentence and forwards them to a sentence Stream. Each sentence is sent through a new socket
- * connection
- *
- * @param stream
- * @throws IOException
- */
- public SocketAdapter(final Stream<Event> stream) throws IOException {
- Thread t = new Thread(new Runnable() {
-
- @Override
- public void run() {
- serverSocket = null;
- Socket connectedSocket;
- BufferedReader in = null;
- try {
- serverSocket = new ServerSocket(12000);
- while (true) {
- connectedSocket = serverSocket.accept();
- in = new BufferedReader(new InputStreamReader(connectedSocket.getInputStream()));
-
- String line = in.readLine();
- System.out.println("read: " + line);
- Event event = new Event();
- event.put("line", String.class, line);
- stream.put(event);
- connectedSocket.close();
- }
-
- } catch (IOException e) {
- e.printStackTrace();
- System.exit(-1);
- } finally {
- if (in != null) {
- try {
- in.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- if (serverSocket != null) {
- try {
- serverSocket.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
- });
- t.start();
-
- }
-
- public void close() {
- if (serverSocket != null) {
- try {
- serverSocket.close();
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/TestApp.java
----------------------------------------------------------------------
diff --git a/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/TestApp.java b/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/TestApp.java
index bbfaf17..389562b 100644
--- a/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/TestApp.java
+++ b/test-apps/simple-deployable-app-1/src/main/java/org/apache/s4/deploy/TestApp.java
@@ -1,8 +1,5 @@
package org.apache.s4.deploy;
-import java.io.IOException;
-import java.util.ArrayList;
-
import org.I0Itec.zkclient.ZkClient;
import org.apache.s4.base.Event;
import org.apache.s4.base.KeyFinder;
@@ -10,10 +7,11 @@ import org.apache.s4.core.App;
import org.apache.s4.core.Stream;
import org.apache.zookeeper.CreateMode;
+import com.google.common.collect.ImmutableList;
+
public class TestApp extends App {
private ZkClient zkClient;
- private SocketAdapter socketAdapter;
@Override
protected void onClose() {
@@ -24,21 +22,12 @@ public class TestApp extends App {
@Override
protected void onInit() {
try {
- try {
- SimplePE prototype = createPE(SimplePE.class);
- Stream<Event> stream = createStream("stream", new KeyFinder<Event>() {
- public java.util.List<String> get(Event event) {
- return new ArrayList<String>() {
- {
- add("line");
- }
- };
- }
- }, prototype);
- socketAdapter = new SocketAdapter(stream);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ SimplePE prototype = createPE(SimplePE.class);
+ Stream<Event> stream = createInputStream("inputStream", new KeyFinder<Event>() {
+ public java.util.List<String> get(Event event) {
+ return ImmutableList.of("line");
+ }
+ }, prototype);
zkClient = new ZkClient("localhost:" + 2181);
if (!zkClient.exists("/s4-test")) {
zkClient.create("/s4-test", null, CreateMode.PERSISTENT);
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopNTopicPE.java
----------------------------------------------------------------------
diff --git a/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopNTopicPE.java b/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopNTopicPE.java
index f6b211c..2218ca5 100644
--- a/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopNTopicPE.java
+++ b/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopNTopicPE.java
@@ -23,6 +23,10 @@ public class TopNTopicPE extends ProcessingElement {
static Logger logger = LoggerFactory.getLogger(TopNTopicPE.class);
Map<String, Integer> countedTopics = Maps.newHashMap();
+ public TopNTopicPE() {
+ // required for checkpointing. Requirement to be lifted in 0.6
+ }
+
public TopNTopicPE(App app) {
super(app);
logger.info("key: [{}]", getId());
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopicCountAndReportPE.java
----------------------------------------------------------------------
diff --git a/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopicCountAndReportPE.java b/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopicCountAndReportPE.java
index 28d87f2..90c3729 100644
--- a/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopicCountAndReportPE.java
+++ b/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TopicCountAndReportPE.java
@@ -9,13 +9,17 @@ import org.slf4j.LoggerFactory;
// keyed by topic name
public class TopicCountAndReportPE extends ProcessingElement {
- Stream<TopicEvent> downStream;
- int threshold = 10;
+ transient Stream<TopicEvent> downStream;
+ transient int threshold = 10;
int count;
boolean firstEvent = true;
static Logger logger = LoggerFactory.getLogger(TopicCountAndReportPE.class);
+ public TopicCountAndReportPE() {
+ // required for checkpointing in S4 0.5. Requirement to be removed in 0.6
+ }
+
public TopicCountAndReportPE(App app) {
super(app);
}
http://git-wip-us.apache.org/repos/asf/incubator-s4/blob/ab1fca5e/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java
----------------------------------------------------------------------
diff --git a/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java b/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java
index 90c3199..fc4e3ae 100644
--- a/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java
+++ b/test-apps/twitter-counter/src/main/java/org/apache/s4/example/twitter/TwitterCounterApp.java
@@ -1,24 +1,20 @@
package org.apache.s4.example.twitter;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import org.I0Itec.zkclient.ZkClient;
import org.apache.s4.base.KeyFinder;
import org.apache.s4.core.App;
import org.apache.s4.core.Stream;
+import org.apache.s4.core.ft.CheckpointingConfig;
+import org.apache.s4.core.ft.CheckpointingConfig.CheckpointingMode;
-public class TwitterCounterApp extends App {
-
- private ZkClient zkClient;
+import com.google.common.collect.ImmutableList;
- private Thread t;
+public class TwitterCounterApp extends App {
@Override
protected void onClose() {
- // TODO Auto-generated method stub
-
}
@Override
@@ -27,31 +23,29 @@ public class TwitterCounterApp extends App {
TopNTopicPE topNTopicPE = createPE(TopNTopicPE.class);
topNTopicPE.setTimerInterval(10, TimeUnit.SECONDS);
+ // we checkpoint this PE every 20s
+ topNTopicPE.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.TIME).frequency(20)
+ .timeUnit(TimeUnit.SECONDS).build());
@SuppressWarnings("unchecked")
Stream<TopicEvent> aggregatedTopicStream = createStream("AggregatedTopicSeen", new KeyFinder<TopicEvent>() {
@Override
public List<String> get(final TopicEvent arg0) {
- return new ArrayList<String>() {
- {
- add("aggregationKey");
- }
- };
+ return ImmutableList.of("aggregationKey");
}
}, topNTopicPE);
TopicCountAndReportPE topicCountAndReportPE = createPE(TopicCountAndReportPE.class);
topicCountAndReportPE.setDownstream(aggregatedTopicStream);
topicCountAndReportPE.setTimerInterval(10, TimeUnit.SECONDS);
+ // we checkpoint instances every 2 events
+ topicCountAndReportPE.setCheckpointingConfig(new CheckpointingConfig.Builder(CheckpointingMode.EVENT_COUNT)
+ .frequency(2).build());
Stream<TopicEvent> topicSeenStream = createStream("TopicSeen", new KeyFinder<TopicEvent>() {
@Override
public List<String> get(final TopicEvent arg0) {
- return new ArrayList<String>() {
- {
- add(arg0.getTopic());
- }
- };
+ return ImmutableList.of(arg0.getTopic());
}
}, topicCountAndReportPE);