You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/06/20 00:02:39 UTC

[1/7] incubator-streams git commit: resolves STREAMS-325 #325

Repository: incubator-streams
Updated Branches:
  refs/heads/master e7bf641d3 -> 39c07f1d6


resolves STREAMS-325 #325


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/8bd22553
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/8bd22553
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/8bd22553

Branch: refs/heads/master
Commit: 8bd225537944305e84a27547b05c24c6c459e7bf
Parents: 6a05779
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Tue May 19 18:00:15 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Tue May 19 18:00:15 2015 -0500

----------------------------------------------------------------------
 streams-monitoring/pom.xml                      | 11 +--
 .../tasks/BroadcastMonitorThread.java           | 81 ++++++++++++--------
 .../monitoring/MonitoringConfiguration.json     | 23 ++++++
 .../tasks/BroadcastMonitorThreadTest.java       | 24 ++++--
 .../local/builders/LocalStreamBuilder.java      | 19 ++++-
 5 files changed, 116 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/pom.xml
----------------------------------------------------------------------
diff --git a/streams-monitoring/pom.xml b/streams-monitoring/pom.xml
index 2f66a07..0e3b86e 100644
--- a/streams-monitoring/pom.xml
+++ b/streams-monitoring/pom.xml
@@ -42,6 +42,11 @@
             <version>${project.version}</version>
         </dependency>
         <dependency>
+            <groupId>org.apache.streams</groupId>
+            <artifactId>streams-config</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpcomponents-core</artifactId>
             <type>pom</type>
@@ -88,11 +93,7 @@
                     <addCompileSourceRoot>true</addCompileSourceRoot>
                     <generateBuilders>true</generateBuilders>
                     <sourcePaths>
-                        <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/Broadcast.json</sourcePath>
-                        <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/StreamsTaskCounterBroadcast.json</sourcePath>
-                        <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/ThroughputQueueBroadcast.json</sourcePath>
-                        <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/DatumStatusCounterBroadcast.json</sourcePath>
-                        <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/MemoryUsageBroadcast.json</sourcePath>
+                        <sourcePath>src/main/jsonschema</sourcePath>
                     </sourcePaths>
                     <outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
                     <targetPackage>org.apache.streams.pojo.json</targetPackage>

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index 19b1067..801d2da 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -21,7 +21,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.collect.Lists;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.jackson.*;
+import org.apache.streams.local.monitoring.MonitoringConfiguration;
 import org.apache.streams.monitoring.persist.MessagePersister;
 import org.apache.streams.monitoring.persist.impl.BroadcastMessagePersister;
 import org.apache.streams.monitoring.persist.impl.LogstashUdpMessagePersister;
@@ -52,38 +55,33 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
     private long DEFAULT_WAIT_TIME = 30000;
     private long waitTime;
     private ObjectMapper objectMapper;
-    private Map<String, Object> streamConfig;
+    private MonitoringConfiguration configuration;
     private URI broadcastURI = null;
     private MessagePersister messagePersister;
     private volatile boolean keepRunning;
 
+    /**
+     * DEPRECATED
+     * Please initialize logging with monitoring object via typesafe
+     * @param streamConfig
+     */
+    @Deprecated
     public BroadcastMonitorThread(Map<String, Object> streamConfig) {
-        keepRunning = true;
-        this.streamConfig = streamConfig;
+        this(new ObjectMapper().convertValue(streamConfig, MonitoringConfiguration.class));
+    }
 
-        LOGGER.info("BroadcastMonitorThread starting" + streamConfig);
+    public BroadcastMonitorThread(MonitoringConfiguration configuration) {
 
-        server = ManagementFactory.getPlatformMBeanServer();
+        this.configuration = configuration;
 
-        setBroadcastURI();
-        setWaitTime();
+        LOGGER.info("BroadcastMonitorThread created");
 
-        if( broadcastURI != null ) {
-            if (broadcastURI.getScheme().equals("http")) {
-                messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
-            } else if (broadcastURI.getScheme().equals("udp")) {
-                messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
-            } else {
-                LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
-                throw new RuntimeException();
-            }
-        } else {
-            messagePersister = new SLF4JMessagePersister();
-        }
+        prepare();
 
         initializeObjectMapper();
 
-        LOGGER.info("BroadcastMonitorThread started");
+        LOGGER.info("BroadcastMonitorThread initialized");
+
     }
 
     /**
@@ -153,12 +151,10 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
      * Go through streams config and set the broadcastURI (if present)
      */
     private void setBroadcastURI() {
-        if(streamConfig != null &&
-                streamConfig.containsKey("broadcastURI") &&
-                streamConfig.get("broadcastURI") != null &&
-                streamConfig.get("broadcastURI") instanceof String) {
+        if(configuration != null &&
+            configuration.getBroadcastURI() != null) {
             try {
-                broadcastURI = new URI(streamConfig.get("broadcastURI").toString());
+                broadcastURI = new URI(configuration.getBroadcastURI());
             } catch (URISyntaxException e) {
                 e.printStackTrace();
             }
@@ -170,12 +166,9 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
      */
     private void setWaitTime() {
         try {
-            if (streamConfig != null &&
-                    streamConfig.containsKey("monitoring_broadcast_interval_ms") &&
-                    streamConfig.get("monitoring_broadcast_interval_ms") != null &&
-                    (streamConfig.get("monitoring_broadcast_interval_ms") instanceof Long ||
-                    streamConfig.get("monitoring_broadcast_interval_ms") instanceof Integer)) {
-                waitTime = Long.parseLong(streamConfig.get("monitoring_broadcast_interval_ms").toString());
+            if (configuration != null &&
+                configuration.getMonitoringBroadcastIntervalMs() != null ) {
+                waitTime = configuration.getMonitoringBroadcastIntervalMs();
             } else {
                 waitTime = DEFAULT_WAIT_TIME;
             }
@@ -189,6 +182,32 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
         }
     }
 
+    public void prepare() {
+
+        keepRunning = true;
+
+        LOGGER.info("BroadcastMonitorThread setup " + configuration);
+
+        server = ManagementFactory.getPlatformMBeanServer();
+
+        setBroadcastURI();
+        setWaitTime();
+
+        if( broadcastURI != null ) {
+            if (broadcastURI.getScheme().equals("http")) {
+                messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
+            } else if (broadcastURI.getScheme().equals("udp")) {
+                messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
+            } else {
+                LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
+                throw new RuntimeException();
+            }
+        } else {
+            messagePersister = new SLF4JMessagePersister();
+        }
+
+    }
+
     public void shutdown() {
         this.keepRunning = false;
         LOGGER.debug("Shutting down BroadcastMonitor Thread");

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
new file mode 100644
index 0000000..2c50cef
--- /dev/null
+++ b/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
@@ -0,0 +1,23 @@
+{
+    "$schema": "http://json-schema.org/draft-03/schema",
+    "$license": [
+        "http://www.apache.org/licenses/LICENSE-2.0"
+    ],
+    "id": "#",
+    "type": "object",
+    "title": "object",
+    "javaType": "org.apache.streams.local.monitoring.MonitoringConfiguration",
+    "javaInterfaces": ["java.io.Serializable"],
+    "description": "Configuration for local runtime monitoring",
+    "properties": {
+        "broadcastURI": {
+            "type": "string",
+            "description": "URI for monitoring statistics"
+        },
+        "monitoring_broadcast_interval_ms": {
+            "type": "integer",
+            "description": "Milliseconds between publish events",
+            "default": 30000
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
index 901888e..7ad315f 100644
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
@@ -18,6 +18,8 @@
 package org.apache.streams.monitoring.tasks;
 
 import com.google.common.collect.Maps;
+import org.apache.streams.local.monitoring.MonitoringConfiguration;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import java.util.Map;
@@ -28,17 +30,29 @@ public class BroadcastMonitorThreadTest {
     private ExecutorService executor;
 
     @Test
-    public void testThreadNullConfig() {
-        BroadcastMonitorThread thread = new BroadcastMonitorThread(null);
+    public void testThreadEmptyBeanConfig() {
+        MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
+        BroadcastMonitorThread thread = new BroadcastMonitorThread(monitoringConfiguration);
+        testThread(thread);
     }
 
     @Test
-    public void testThread() {
+    public void testThreadEmptyMapConfig() {
+        Map<String, Object> map = Maps.newHashMap();
+        BroadcastMonitorThread thread = new BroadcastMonitorThread(map);
+        testThread(thread);
+    }
+
+    @Test
+    public void testThreadFakeMapConfig() {
         Map<String, Object> config = Maps.newHashMap();
         config.put("broadcastURI", "http://fakeurl.com/fake");
-
         BroadcastMonitorThread thread = new BroadcastMonitorThread(config);
-        thread.setDefaultWaitTime(30000L);
+        testThread(thread);
+    }
+
+    public void testThread(BroadcastMonitorThread thread) {
+        thread.setDefaultWaitTime(3000L);
         long testRunLength = thread.getDefaultWaitTime() * 1;
         executor = Executors.newFixedThreadPool(1);
         executor.submit(thread);

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 601c6b7..f6d8f23 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -19,9 +19,12 @@
 package org.apache.streams.local.builders;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.core.*;
 import org.apache.streams.local.counters.StreamsTaskCounter;
 import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
+import org.apache.streams.local.monitoring.MonitoringConfiguration;
 import org.apache.streams.local.queues.ThroughputQueue;
 import org.apache.streams.local.tasks.*;
 import org.apache.streams.monitoring.tasks.BroadcastMonitorThread;
@@ -120,7 +123,21 @@ public class LocalStreamBuilder implements StreamBuilder {
             this.streamConfig.put(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
         }
         this.useDeprecatedMonitors = false;
-        this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
+
+        /* for backward-compatibility with streamConfig */
+        MonitoringConfiguration monitoringConfiguration;
+        if( StreamsConfigurator.getConfig().hasPath("monitoring") ) {
+            monitoringConfiguration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration("monitoring");
+        } else {
+            monitoringConfiguration = new MonitoringConfiguration();
+        }
+        if( this.streamConfig != null &&
+            this.streamConfig.containsKey(BROADCAST_KEY)) {
+            monitoringConfiguration.setBroadcastURI(this.streamConfig.get(BROADCAST_KEY).toString());
+            if( this.streamConfig.containsKey(BROADCAST_INTERVAL_KEY))
+                monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong(this.streamConfig.get(BROADCAST_INTERVAL_KEY).toString()));
+        }
+        this.broadcastMonitor = new BroadcastMonitorThread(monitoringConfiguration);
 
         this.futures = new HashMap<>();
     }


[4/7] incubator-streams git commit: completed, all tests passing resolves #226 resolves #294 resolves #325

Posted by sb...@apache.org.
completed, all tests passing
resolves #226
resolves #294
resolves #325


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4081987c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4081987c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4081987c

Branch: refs/heads/master
Commit: 4081987c3636886457f0dd7eff731e88c8954471
Parents: 40b9b1c
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Mon May 25 18:51:10 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Mon May 25 18:51:10 2015 -0500

----------------------------------------------------------------------
 .../main/jsonschema/StreamsConfiguration.json   | 20 +++---
 .../tasks/BroadcastMonitorThread.java           | 11 ++-
 .../tasks/BroadcastMonitorThreadTest.java       |  5 +-
 .../local/builders/LocalStreamBuilder.java      | 75 +++++++++++---------
 .../streams/local/tasks/BaseStreamsTask.java    | 22 +++---
 .../local/tasks/StreamsPersistWriterTask.java   | 17 ++---
 .../local/tasks/StreamsProcessorTask.java       | 15 +---
 .../local/tasks/StreamsProviderTask.java        | 13 ++--
 .../apache/streams/local/tasks/StreamsTask.java | 10 ---
 .../streams/local/tasks/BasicTasksTest.java     | 60 +++++-----------
 10 files changed, 107 insertions(+), 141 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-config/src/main/jsonschema/StreamsConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-config/src/main/jsonschema/StreamsConfiguration.json b/streams-config/src/main/jsonschema/StreamsConfiguration.json
index bfff501..d331df2 100644
--- a/streams-config/src/main/jsonschema/StreamsConfiguration.json
+++ b/streams-config/src/main/jsonschema/StreamsConfiguration.json
@@ -8,6 +8,11 @@
     "javaType" : "org.apache.streams.config.StreamsConfiguration",
     "javaInterfaces": ["java.io.Serializable"],
     "properties": {
+        "batchFrequencyMs": {
+            "type": "integer",
+            "description": "Frequency of messaging between components",
+            "default": 5000
+        },
         "batchSize": {
             "type" : "integer",
             "description" : "Default max batch size for batch-based persist readers and writers",
@@ -23,21 +28,16 @@
             "description" : "Default parallelism for processors",
             "default": 1
         },
-        "queueSize": {
-            "type" : "integer",
-            "description" : "Size of inter-component buffers",
-            "default": 1000
-        },
-        "batchFrequencyMs": {
-            "type": "integer",
-            "description": "Frequency of messaging between components",
-            "default": 5000
-        },
         "providerTimeoutMs": {
             "type": "integer",
             "description": "Kill the stream if all providers supply nothing for this long.",
             "default": 1440000
         },
+        "queueSize": {
+            "type" : "integer",
+            "description" : "Size of inter-component buffers",
+            "default": 1000
+        },
         "startedAt": {
             "type": "integer",
             "description": "Timestamp of beginning of stream execution",

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index 5d0377a..b9db630 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.google.common.collect.Lists;
 import org.apache.streams.config.ComponentConfigurator;
 import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.jackson.*;
 import org.apache.streams.local.monitoring.MonitoringConfiguration;
 import org.apache.streams.monitoring.persist.MessagePersister;
@@ -55,6 +56,7 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
     private long DEFAULT_WAIT_TIME = 30000;
     private long waitTime;
     private ObjectMapper objectMapper;
+    private StreamsConfiguration streamConfig;
     private MonitoringConfiguration configuration;
     private URI broadcastURI = null;
     private MessagePersister messagePersister;
@@ -67,11 +69,16 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
      */
     @Deprecated
     public BroadcastMonitorThread(Map<String, Object> streamConfig) {
-        this(new ObjectMapper().convertValue(streamConfig, MonitoringConfiguration.class));
+        this(new ObjectMapper().convertValue(streamConfig, StreamsConfiguration.class));
     }
 
-    public BroadcastMonitorThread(MonitoringConfiguration configuration) {
+    public BroadcastMonitorThread(StreamsConfiguration streamConfig) {
+        this(streamConfig, new ObjectMapper().convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
+    }
+
+    public BroadcastMonitorThread(StreamsConfiguration streamConfig, MonitoringConfiguration configuration) {
 
+        this.streamConfig = streamConfig;
         this.configuration = configuration;
 
         LOGGER.info("BroadcastMonitorThread created");

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
index 7ad315f..427757b 100644
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
@@ -18,6 +18,7 @@
 package org.apache.streams.monitoring.tasks;
 
 import com.google.common.collect.Maps;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.local.monitoring.MonitoringConfiguration;
 import org.junit.Ignore;
 import org.junit.Test;
@@ -31,8 +32,8 @@ public class BroadcastMonitorThreadTest {
 
     @Test
     public void testThreadEmptyBeanConfig() {
-        MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
-        BroadcastMonitorThread thread = new BroadcastMonitorThread(monitoringConfiguration);
+        StreamsConfiguration streamsConfiguration = new StreamsConfiguration();
+        BroadcastMonitorThread thread = new BroadcastMonitorThread(streamsConfiguration);
         testThread(thread);
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 59d60df..4175e10 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -18,8 +18,10 @@
 
 package org.apache.streams.local.builders;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.core.*;
 import org.apache.streams.local.LocalRuntimeConfiguration;
@@ -74,16 +76,16 @@ public class LocalStreamBuilder implements StreamBuilder {
     private boolean useDeprecatedMonitors;
 
     /**
-     * Creates a local stream builder with no config object and default maximum internal queue size of 500
+     * Creates a local stream builder with all configuration resolved by typesafe
      */
-    public LocalStreamBuilder(){
-        this(DEFAULT_QUEUE_SIZE, null);
+    public LocalStreamBuilder() {
+        this(new ObjectMapper().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class));
     }
 
     /**
      * Creates a local stream builder with a config object and default maximum internal queue size of 500
      * @param streamConfig
-     * @deprecated use LocalRuntimeConfiguration constructor instread
+     * @deprecated use LocalRuntimeConfiguration constructor instead
      */
     @Deprecated
     public LocalStreamBuilder(Map<String, Object> streamConfig) {
@@ -94,7 +96,10 @@ public class LocalStreamBuilder implements StreamBuilder {
      * Creates a local stream builder with no config object. If maxQueueCapacity is less than 1 the queue is
      * unbounded.
      * @param maxQueueCapacity
+     *
+     * @deprecated use LocalRuntimeConfiguration constructor instead
      */
+    @Deprecated
     public LocalStreamBuilder(int maxQueueCapacity) {
         this(maxQueueCapacity, null);
     }
@@ -102,23 +107,41 @@ public class LocalStreamBuilder implements StreamBuilder {
     /**
      * Creates a local stream builder with a config object. If maxQueueCapacity is less than 1 the queue is
      * unbounded.
+     *
      * @param maxQueueCapacity
      * @param streamConfig
+     *
+     * @deprecated use LocalRuntimeConfiguration constructor instead
      */
+    @Deprecated
     public LocalStreamBuilder(int maxQueueCapacity, Map<String, Object> streamConfig) {
-        LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
-        for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
-            config.setAdditionalProperty(item.getKey(), item.getValue());
+        this(new LocalRuntimeConfiguration());
+        this.streamConfig.setQueueSize(new Long(maxQueueCapacity));
+        if( streamConfig != null && streamConfig.get(LocalStreamBuilder.TIMEOUT_KEY) != null )
+            this.streamConfig.setProviderTimeoutMs(new Long((Integer) (streamConfig.get(LocalStreamBuilder.TIMEOUT_KEY))));
+        if( streamConfig != null && streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) != null )
+            this.streamConfig.setIdentifier((String)streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY));
+        if( streamConfig != null && streamConfig.get(LocalStreamBuilder.BROADCAST_KEY) != null ) {
+            MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
+            monitoringConfiguration.setBroadcastURI((String)streamConfig.get(LocalStreamBuilder.BROADCAST_KEY));
+            if(streamConfig.get(LocalStreamBuilder.BROADCAST_INTERVAL_KEY) != null)
+                monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong((String)streamConfig.get(LocalStreamBuilder.BROADCAST_INTERVAL_KEY)));
+            this.streamConfig.setMonitoring(monitoringConfiguration);
         }
-        config.setBatchSize(new Long(maxQueueCapacity));
     }
 
     public LocalStreamBuilder(LocalRuntimeConfiguration streamConfig) {
+        this.streamConfig = streamConfig;
         this.providers = new HashMap<String, StreamComponent>();
         this.components = new HashMap<String, StreamComponent>();
-        this.streamConfig = streamConfig;
         this.totalTasks = 0;
         this.monitorTasks = 0;
+        this.futures = new HashMap<>();
+    }
+
+    public void prepare() {
+        this.streamIdentifier = streamConfig.getIdentifier();
+        this.streamConfig.setStartedAt(startedAt.getMillis());
         final LocalStreamBuilder self = this;
         this.shutdownHook = new Thread() {
             @Override
@@ -127,29 +150,8 @@ public class LocalStreamBuilder implements StreamBuilder {
                 self.stopInternal(true);
             }
         };
-
-        setStreamIdentifier();
-        if(this.streamConfig != null) {
-            this.streamConfig.setAdditionalProperty(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
-        }
         this.useDeprecatedMonitors = false;
-
-        /* for backward-compatibility with streamConfig */
-        MonitoringConfiguration monitoringConfiguration;
-        if( StreamsConfigurator.getConfig().hasPath("monitoring") ) {
-            monitoringConfiguration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration("monitoring");
-        } else {
-            monitoringConfiguration = new MonitoringConfiguration();
-        }
-        if( this.streamConfig != null &&
-            this.streamConfig.containsKey(BROADCAST_KEY)) {
-            monitoringConfiguration.setBroadcastURI(this.streamConfig.get(BROADCAST_KEY).toString());
-            if( this.streamConfig.containsKey(BROADCAST_INTERVAL_KEY))
-                monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong(this.streamConfig.get(BROADCAST_INTERVAL_KEY).toString()));
-        }
-        this.broadcastMonitor = new BroadcastMonitorThread(monitoringConfiguration);
-
-        this.futures = new HashMap<>();
+        this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
     }
 
     public void setUseDeprecatedMonitors(boolean useDeprecatedMonitors) {
@@ -225,6 +227,7 @@ public class LocalStreamBuilder implements StreamBuilder {
      */
     @Override
     public void start() {
+        prepare();
         attachShutdownHandler();
         boolean isRunning = true;
         this.executor = new ShutdownStreamOnUnhandleThrowableThreadPoolExecutor(this.totalTasks, this);
@@ -465,8 +468,14 @@ public class LocalStreamBuilder implements StreamBuilder {
         return streamConfig.getProviderTimeoutMs().intValue();
     }
 
-    private void setStreamIdentifier() {
-        this.streamIdentifier = streamConfig.getIdentifier();
+    private LocalRuntimeConfiguration convertConfiguration(Map<String, Object> streamConfig) {
+        LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
+        if( streamConfig != null ) {
+            for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
+                config.setAdditionalProperty(item.getKey(), item.getValue());
+            }
+        }
+        return config;
     }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
index 3daa110..8bc7769 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
@@ -22,12 +22,14 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.core.StreamsDatum;
 import org.apache.streams.jackson.StreamsJacksonMapper;
 import org.apache.streams.local.builders.LocalStreamBuilder;
 import org.apache.streams.pojo.json.Activity;
 import org.apache.streams.util.ComponentUtils;
 import org.apache.streams.util.SerializationUtil;
+import org.joda.time.DateTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -49,15 +51,14 @@ public abstract class BaseStreamsTask implements StreamsTask {
     private ObjectMapper mapper;
     protected StreamsConfiguration streamConfig;
 
-    private long startedAt;
-    private String streamIdentifier;
-
     public BaseStreamsTask(StreamsConfiguration config) {
         this.mapper = StreamsJacksonMapper.getInstance();
         this.mapper.registerSubtypes(Activity.class);
-        this.streamConfig = config;
+        if( config != null )
+            this.streamConfig = config;
+        else
+            this.streamConfig = StreamsConfigurator.detectConfiguration();
 
-        setStreamIdentifier();
         setStartedAt();
     }
 
@@ -202,20 +203,15 @@ public abstract class BaseStreamsTask implements StreamsTask {
     }
 
     public long getStartedAt() {
-        return startedAt;
+        return streamConfig.getStartedAt();
     }
 
     public void setStartedAt() {
-        this.startedAt = streamConfig.getStartedAt();
+        streamConfig.setStartedAt(DateTime.now().getMillis());
     }
 
     public String getStreamIdentifier() {
-        return streamIdentifier;
-    }
-
-    public void setStreamIdentifier() {
-        this.streamIdentifier = streamConfig.getIdentifier();
+        return streamConfig.getIdentifier();
     }
 
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index d6bf2ec..a529c69 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -39,7 +39,6 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
     private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPersistWriterTask.class);
 
     private StreamsPersistWriter writer;
-    private long sleepTime = DEFAULT_SLEEP_TIME_MS * 10;
     private AtomicBoolean keepRunning;
     private StreamsConfiguration streamConfig;
     private BlockingQueue<StreamsDatum> inQueue;
@@ -60,22 +59,18 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
      * @param writer writer to execute in task
      */
     public StreamsPersistWriterTask(StreamsPersistWriter writer) {
-        this(writer, DEFAULT_SLEEP_TIME_MS, null);
-    }
-
-    public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
-        this(writer, DEFAULT_SLEEP_TIME_MS, streamConfig);
+        this(writer, null);
     }
 
     /**
      *
      * @param writer writer to execute in task
-     * @param sleepTime time to sleep when inbound queue is empty.
+     * @param streamConfig stream config
      */
-    public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime, StreamsConfiguration streamConfig) {
+    public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
         super(streamConfig);
+        this.streamConfig = super.streamConfig;
         this.writer = writer;
-        this.sleepTime = sleepTime;
         this.keepRunning = new AtomicBoolean(true);
         this.isRunning = new AtomicBoolean(true);
         this.blocked = new AtomicBoolean(false);
@@ -141,11 +136,11 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
                     LOGGER.debug("Received null StreamsDatum @ writer : {}", this.writer.getClass().getName());
                 }
             }
-            Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
+            Uninterruptibles.sleepUninterruptibly(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
         } catch(Exception e) {
             LOGGER.error("Failed to execute Persist Writer {}",this.writer.getClass().getSimpleName(), e);
         } finally {
-            Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
+            Uninterruptibles.sleepUninterruptibly(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
             this.writer.cleanUp();
             this.isRunning.set(false);
         }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index 5e04b2f..72b3eb6 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -40,7 +40,6 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
 
 
     private StreamsProcessor processor;
-    private long sleepTime;
     private AtomicBoolean keepRunning;
     private StreamsConfiguration streamConfig;
     private BlockingQueue<StreamsDatum> inQueue;
@@ -60,25 +59,17 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
      * @param processor process to run in task
      */
     public StreamsProcessorTask(StreamsProcessor processor) {
-        this(processor, DEFAULT_SLEEP_TIME_MS, null);
+        this(processor, new StreamsConfiguration());
     }
 
     /**
-     *
      * @param processor
      * @param streamConfig
      */
-    public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) { this(processor, DEFAULT_SLEEP_TIME_MS, streamConfig); }
-
-    /**
-     *
-     * @param processor processor to run in task
-     * @param sleepTime time to sleep when incoming queue is empty
-     */
-    public StreamsProcessorTask(StreamsProcessor processor, long sleepTime, StreamsConfiguration streamConfig) {
+    public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) {
         super(streamConfig);
+        this.streamConfig = super.streamConfig;
         this.processor = processor;
-        this.sleepTime = sleepTime;
         this.keepRunning = new AtomicBoolean(true);
         this.isRunning = new AtomicBoolean(true);
         this.blocked = new AtomicBoolean(true);

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index cb36a24..53fe7c6 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -77,13 +77,14 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
      */
     public StreamsProviderTask(StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
         super(streamConfig);
+        streamConfig = super.streamConfig;
         this.provider = provider;
         if( perpetual )
             this.type = Type.PERPETUAL;
         else
             this.type = Type.READ_CURRENT;
-        this.timeout = DEFAULT_TIMEOUT_MS;
-        this.sleepTime = DEFAULT_SLEEP_TIME_MS;
+        this.timeout = super.streamConfig.getProviderTimeoutMs().intValue();
+        this.sleepTime = streamConfig.getBatchFrequencyMs();
     }
 
     /**
@@ -96,8 +97,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
         this.provider = provider;
         this.type = Type.READ_NEW;
         this.sequence = sequence;
-        this.timeout = DEFAULT_TIMEOUT_MS;
-        this.sleepTime = DEFAULT_SLEEP_TIME_MS;
+        this.timeout = streamConfig.getProviderTimeoutMs().intValue();
+        this.sleepTime = streamConfig.getBatchFrequencyMs();
     }
 
     /**
@@ -113,8 +114,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
         this.dateRange = new DateTime[2];
         this.dateRange[START] = start;
         this.dateRange[END] = end;
-        this.timeout = DEFAULT_TIMEOUT_MS;
-        this.sleepTime = DEFAULT_SLEEP_TIME_MS;
+        this.timeout = streamConfig.getProviderTimeoutMs().intValue();
+        this.sleepTime = streamConfig.getBatchFrequencyMs();
     }
 
     public void setTimeout(int timeout) {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
index e98dc0b..5c14c1f 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -32,16 +32,6 @@ import java.util.concurrent.BlockingQueue;
  * in local mode.
  */
 public interface StreamsTask extends Runnable{
-    /**
-     * Represents the default time to sleep a task before polling for the next set of work items
-     */
-    static final long DEFAULT_SLEEP_TIME_MS = 5000;
-
-    /**
-     * Represents the default amount of time to wait for new data to flow through the system before assuming it should
-     * shut down
-     */
-    static final int DEFAULT_TIMEOUT_MS = 100000; //Will result in a default timeout of 1 minute if used
 
     /**
      * Informs the task to stop. Tasks may or may not try to empty its inbound queue before halting.

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
index 7671a2d..38e948e 100644
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@ -18,6 +18,7 @@
 
 package org.apache.streams.local.tasks;
 
+import com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.streams.core.StreamsDatum;
 import org.apache.streams.local.counters.DatumStatusCounter;
 import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -74,12 +75,7 @@ public class BasicTasksTest {
         service.submit(task);
         int attempts = 0;
         while(outQueue.size() != numMessages) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                Thread.currentThread().interrupt();
-            }
-            ++attempts;
+            Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
             if(attempts == 10) {
                 fail("Provider task failed to output "+numMessages+" in a timely fashion.");
             }
@@ -90,7 +86,7 @@ public class BasicTasksTest {
                 service.shutdownNow();
                 fail("Service did not terminate.");
             }
-            assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+            assertTrue("Task should have completed running in allotted time.", service.isTerminated());
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt();
         }
@@ -112,11 +108,7 @@ public class BasicTasksTest {
         service.submit(task);
         int attempts = 0;
         while(inQueue.size() != 0 && outQueue.size() != numMessages) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                //Ignore
-            }
+            Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
             ++attempts;
             if(attempts == 10) {
                 fail("Processor task failed to output "+numMessages+" in a timely fashion.");
@@ -129,9 +121,9 @@ public class BasicTasksTest {
                 service.shutdownNow();
                 fail("Service did not terminate.");
             }
-            assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+            assertTrue("Task should have completed running in allotted time.", service.isTerminated());
         } catch (InterruptedException e) {
-            fail("Test Interupted.");
+            fail("Test Interrupted.");
         }
         assertEquals(numMessages, processor.getMessageCount());
         assertEquals(numMessages, counter.getNumReceived());
@@ -163,11 +155,7 @@ public class BasicTasksTest {
         service.submit(task);
         int attempts = 0;
         while(inQueue.size() != 0 ) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                //Ignore
-            }
+            Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
             ++attempts;
             if(attempts == 10) {
                 fail("Processor task failed to output "+numMessages+" in a timely fashion.");
@@ -180,9 +168,9 @@ public class BasicTasksTest {
                 service.shutdownNow();
                 fail("Service did not terminate.");
             }
-            assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+            assertTrue("Task should have completed running in allotted time.", service.isTerminated());
         } catch (InterruptedException e) {
-            fail("Test Interupted.");
+            fail("Test Interrupted.");
         }
         assertEquals(numMessages, writer.getDatumsCounted());
         assertEquals(numMessages, counter.getNumReceived());
@@ -205,11 +193,7 @@ public class BasicTasksTest {
         service.submit(task);
         int attempts = 0;
         while(outQueue.size() != incoming * numMessages ) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                //Ignore
-            }
+            Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
             ++attempts;
             if(attempts == 10) {
                 assertEquals("Processor task failed to output " + (numMessages * incoming) + " in a timely fashion.", (numMessages * incoming), outQueue.size());
@@ -222,9 +206,9 @@ public class BasicTasksTest {
                 service.shutdownNow();
                 fail("Service did not terminate.");
             }
-            assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+            assertTrue("Task should have completed running in allotted time.", service.isTerminated());
         } catch (InterruptedException e) {
-            fail("Test Interupted.");
+            fail("Test Interrupted.");
         }
     }
 
@@ -244,11 +228,7 @@ public class BasicTasksTest {
         service.submit(task);
         int attempts = 0;
         while(inQueue.size() != 0 ) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                //Ignore
-            }
+            Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
             ++attempts;
             if(attempts == 10) {
                 assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
@@ -262,9 +242,9 @@ public class BasicTasksTest {
                 service.shutdownNow();
                 fail("Service did not terminate.");
             }
-            assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+            assertTrue("Task should have completed running in allotted time.", service.isTerminated());
         } catch (InterruptedException e) {
-            fail("Test Interupted.");
+            fail("Test Interrupted.");
         }
         assertEquals(numMessages, processor.getMessageCount());
         assertEquals(numMessages, outQueue1.size());
@@ -286,11 +266,7 @@ public class BasicTasksTest {
         service.submit(task);
         int attempts = 0;
         while(inQueue.size() != 0 ) {
-            try {
-                Thread.sleep(500);
-            } catch (InterruptedException e) {
-                //Ignore
-            }
+            Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
             ++attempts;
             if(attempts == 10) {
                 assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
@@ -304,9 +280,9 @@ public class BasicTasksTest {
                 service.shutdownNow();
                 fail("Service did not terminate.");
             }
-            assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+            assertTrue("Task should have completed running in allotted time.", service.isTerminated());
         } catch (InterruptedException e) {
-            fail("Test Interupted.");
+            fail("Test Interrupted.");
         }
         assertEquals(numMessages, processor.getMessageCount());
         assertEquals(numMessages, outQueue1.size());


[6/7] incubator-streams git commit: improvements and new test for STREAMS-325

Posted by sb...@apache.org.
improvements and new test for STREAMS-325


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/94c3fd5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/94c3fd5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/94c3fd5e

Branch: refs/heads/master
Commit: 94c3fd5ea577f0710f61e7eb4c5c9af57ec33001
Parents: ebeaa16
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Fri Jun 12 11:29:32 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Fri Jun 12 11:53:14 2015 -0500

----------------------------------------------------------------------
 .../tasks/BroadcastMonitorThread.java           | 96 ++++++--------------
 .../tasks/BroadcastMonitorThreadTest.java       | 15 ++-
 .../local/builders/LocalStreamBuilder.java      |  2 +-
 3 files changed, 44 insertions(+), 69 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c3fd5e/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index b9db630..20c605e 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -53,15 +53,13 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
     private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BroadcastMonitorThread.class);
     private static MBeanServer server;
 
-    private long DEFAULT_WAIT_TIME = 30000;
-    private long waitTime;
-    private ObjectMapper objectMapper;
-    private StreamsConfiguration streamConfig;
     private MonitoringConfiguration configuration;
     private URI broadcastURI = null;
     private MessagePersister messagePersister;
     private volatile boolean keepRunning;
 
+    private static ObjectMapper objectMapper = StreamsJacksonMapper.getInstance();
+
     /**
      * DEPRECATED
      * Please initialize logging with monitoring object via typesafe
@@ -69,24 +67,25 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
      */
     @Deprecated
     public BroadcastMonitorThread(Map<String, Object> streamConfig) {
-        this(new ObjectMapper().convertValue(streamConfig, StreamsConfiguration.class));
+        this(objectMapper.convertValue(streamConfig, MonitoringConfiguration.class));
     }
 
     public BroadcastMonitorThread(StreamsConfiguration streamConfig) {
-        this(streamConfig, new ObjectMapper().convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
+        this(objectMapper.convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
     }
 
-    public BroadcastMonitorThread(StreamsConfiguration streamConfig, MonitoringConfiguration configuration) {
+    public BroadcastMonitorThread(MonitoringConfiguration configuration) {
 
-        this.streamConfig = streamConfig;
         this.configuration = configuration;
+        if( this.configuration == null )
+            this.configuration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().atPath("monitoring"));
 
         LOGGER.info("BroadcastMonitorThread created");
 
-        prepare();
-
         initializeObjectMapper();
 
+        prepare();
+
         LOGGER.info("BroadcastMonitorThread initialized");
 
     }
@@ -97,7 +96,6 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
      * POJOs which are generated from JSON schemas
      */
     private void initializeObjectMapper() {
-        objectMapper = StreamsJacksonMapper.getInstance();
         SimpleModule simpleModule = new SimpleModule();
 
         simpleModule.addDeserializer(MemoryUsageBroadcast.class, new MemoryUsageDeserializer());
@@ -142,7 +140,7 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
                 }
 
                 messagePersister.persistMessages(messages);
-                Thread.sleep(waitTime);
+                Thread.sleep(configuration.getMonitoringBroadcastIntervalMs());
             } catch (InterruptedException e) {
                 LOGGER.debug("Broadcast Monitor Interrupted!");
                 Thread.currentThread().interrupt();
@@ -154,60 +152,33 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
         }
     }
 
-    /**
-     * Go through streams config and set the broadcastURI (if present)
-     */
-    private void setBroadcastURI() {
-        if(configuration != null &&
-            configuration.getBroadcastURI() != null) {
-            try {
-                broadcastURI = new URI(configuration.getBroadcastURI());
-            } catch (URISyntaxException e) {
-                e.printStackTrace();
-            }
-        }
-    }
-
-    /**
-     * Go through streams config and set the thread's wait time (if present)
-     */
-    private void setWaitTime() {
-        try {
-            if (configuration != null &&
-                configuration.getMonitoringBroadcastIntervalMs() != null ) {
-                waitTime = configuration.getMonitoringBroadcastIntervalMs();
-            } else {
-                waitTime = DEFAULT_WAIT_TIME;
-            }
-
-            //Shutdown
-            if(waitTime == -1) {
-                this.keepRunning = false;
-            }
-        } catch (Exception e) {
-            LOGGER.error("Exception while trying to set default broadcast thread wait time: {}", e);
-        }
-    }
-
     public void prepare() {
 
         keepRunning = true;
 
-        LOGGER.info("BroadcastMonitorThread setup " + configuration);
+        LOGGER.info("BroadcastMonitorThread setup " + this.configuration);
 
         server = ManagementFactory.getPlatformMBeanServer();
 
-        setBroadcastURI();
-        setWaitTime();
+        if (this.configuration != null) {
+
+            try {
+                broadcastURI = new URI(configuration.getBroadcastURI());
+            } catch (Exception e) {
+                LOGGER.error("invalid URI: ", e);
+            }
 
-        if( broadcastURI != null ) {
-            if (broadcastURI.getScheme().equals("http")) {
-                messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
-            } else if (broadcastURI.getScheme().equals("udp")) {
-                messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
+            if (broadcastURI != null) {
+                if (broadcastURI.getScheme().equals("http")) {
+                    messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
+                } else if (broadcastURI.getScheme().equals("udp")) {
+                    messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
+                } else {
+                    LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
+                    throw new RuntimeException();
+                }
             } else {
-                LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
-                throw new RuntimeException();
+                messagePersister = new SLF4JMessagePersister();
             }
         } else {
             messagePersister = new SLF4JMessagePersister();
@@ -221,18 +192,11 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
     }
 
     public String getBroadcastURI() {
-        return broadcastURI.toString();
+        return configuration.getBroadcastURI();
     }
 
     public long getWaitTime() {
-        return waitTime;
-    }
-
-    public long getDefaultWaitTime() {
-        return DEFAULT_WAIT_TIME;
+        return configuration.getMonitoringBroadcastIntervalMs();
     }
 
-    public void setDefaultWaitTime(long defaultWaitTime) {
-        this.DEFAULT_WAIT_TIME = defaultWaitTime;
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c3fd5e/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
index 427757b..a959bd2 100644
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
@@ -52,9 +52,20 @@ public class BroadcastMonitorThreadTest {
         testThread(thread);
     }
 
+    @Test
+    public void testThreadStreamsConfig() {
+
+        StreamsConfiguration streams = new StreamsConfiguration();
+        MonitoringConfiguration monitoring = new MonitoringConfiguration();
+        monitoring.setBroadcastURI("http://fakeurl.com/fake");
+        monitoring.setMonitoringBroadcastIntervalMs(30000L);
+        streams.setAdditionalProperty("monitoring", monitoring);
+        BroadcastMonitorThread thread = new BroadcastMonitorThread(streams);
+        testThread(thread);
+    }
+
     public void testThread(BroadcastMonitorThread thread) {
-        thread.setDefaultWaitTime(3000L);
-        long testRunLength = thread.getDefaultWaitTime() * 1;
+        long testRunLength = thread.getWaitTime() * 1;
         executor = Executors.newFixedThreadPool(1);
         executor.submit(thread);
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c3fd5e/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 4175e10..4cf078e 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -151,7 +151,7 @@ public class LocalStreamBuilder implements StreamBuilder {
             }
         };
         this.useDeprecatedMonitors = false;
-        this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
+        this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig.getMonitoring());
     }
 
     public void setUseDeprecatedMonitors(boolean useDeprecatedMonitors) {


[5/7] incubator-streams git commit: fix one reference to removed config field adapts dropwizard runtime configuration

Posted by sb...@apache.org.
fix one reference to removed config field
adapts dropwizard runtime configuration


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ebeaa162
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ebeaa162
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ebeaa162

Branch: refs/heads/master
Commit: ebeaa1622a1e1235d17a59d20ab0dabe1456fe7e
Parents: 4081987
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Mon May 25 22:08:49 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Mon May 25 22:08:49 2015 -0500

----------------------------------------------------------------------
 .../apache/streams/dropwizard/StreamDropwizardBuilder.java   | 8 ++++++++
 .../org/apache/streams/dropwizard/StreamsApplication.java    | 6 +-----
 .../org/apache/streams/local/tasks/StreamsMergeTask.java     | 8 ++------
 3 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ebeaa162/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
index d409a4a..0fcc4eb 100644
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
+++ b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
@@ -18,8 +18,12 @@
 
 package org.apache.streams.dropwizard;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.core.StreamBuilder;
 import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.local.LocalRuntimeConfiguration;
 import org.apache.streams.local.builders.LocalStreamBuilder;
 
 import java.util.Map;
@@ -35,6 +39,10 @@ public class StreamDropwizardBuilder extends LocalStreamBuilder implements Strea
         super();
     }
 
+    public StreamDropwizardBuilder(StreamsConfiguration streamConfig) {
+        super(new ObjectMapper().convertValue(streamConfig, LocalRuntimeConfiguration.class));
+    }
+
     public StreamDropwizardBuilder(Map<String, Object> streamConfig) {
         super(streamConfig);
     }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ebeaa162/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
index 991d900..463320c 100644
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
+++ b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
@@ -145,11 +145,7 @@ public class StreamsApplication extends Application<StreamsDropwizardConfigurati
 
     public StreamBuilder setup(StreamsConfiguration streamsConfiguration, Set<StreamsProvider> resourceProviders) {
 
-        Map<String, Object> streamConfig = Maps.newHashMap();
-        streamConfig.put(LocalStreamBuilder.TIMEOUT_KEY, streamsConfiguration.getTimeoutMs().intValue());
-        if(! Strings.isNullOrEmpty(streamsConfiguration.getBroadcastURI()) ) streamConfig.put("broadcastURI", streamsConfiguration.getBroadcastURI());
-        if(streamsConfiguration.getBroadcastIntervalMs() != null ) streamConfig.put("monitoring_broadcast_interval_ms", streamsConfiguration.getBroadcastIntervalMs().intValue());
-        StreamBuilder builder = new StreamDropwizardBuilder(1000, streamConfig);
+        StreamBuilder builder = new StreamDropwizardBuilder(streamsConfiguration);
 
         List<String> providers = new ArrayList<>();
         for( StreamsProvider provider: resourceProviders) {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ebeaa162/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
index 49cb074..69cd5a5 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -38,14 +38,10 @@ public class StreamsMergeTask extends BaseStreamsTask {
     private long sleepTime;
 
     public StreamsMergeTask() {
-        this(DEFAULT_SLEEP_TIME_MS, null);
+        this(null);
     }
 
-    public StreamsMergeTask(long sleepTime) {
-        this(sleepTime, null);
-    }
-
-    public StreamsMergeTask(long sleepTime, StreamsConfiguration streamConfig) {
+    public StreamsMergeTask(StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.sleepTime = sleepTime;
         this.keepRunning = new AtomicBoolean(true);


[7/7] incubator-streams git commit: Merge commit '94c3fd5ea577f0710f61e7eb4c5c9af57ec33001'

Posted by sb...@apache.org.
Merge commit '94c3fd5ea577f0710f61e7eb4c5c9af57ec33001'

* commit '94c3fd5ea577f0710f61e7eb4c5c9af57ec33001':
  improvements and new test for STREAMS-325
  fix one reference to removed config field adapts dropwizard runtime configuration
  completed, all tests passing resolves #226 resolves #294 resolves #325
  initial refactoring
  resolves STREAMS-325 #325


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/39c07f1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/39c07f1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/39c07f1d

Branch: refs/heads/master
Commit: 39c07f1d64757626acf496028a0367fa5d347ff0
Parents: e7bf641 94c3fd5
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Fri Jun 19 16:44:46 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Fri Jun 19 16:44:46 2015 -0500

----------------------------------------------------------------------
 .../main/jsonschema/StreamsConfiguration.json   |  38 ++++--
 streams-monitoring/pom.xml                      |  11 +-
 .../tasks/BroadcastMonitorThread.java           | 124 +++++++++----------
 .../monitoring/MonitoringConfiguration.json     |  23 ++++
 .../tasks/BroadcastMonitorThreadTest.java       |  38 +++++-
 .../dropwizard/StreamDropwizardBuilder.java     |   8 ++
 .../streams/dropwizard/StreamsApplication.java  |   6 +-
 .../local/builders/LocalStreamBuilder.java      |  75 +++++++----
 .../streams/local/builders/StreamComponent.java |  13 +-
 .../streams/local/tasks/BaseStreamsTask.java    |  41 ++----
 .../streams/local/tasks/StreamsMergeTask.java   |  11 +-
 .../local/tasks/StreamsPersistWriterTask.java   |  22 ++--
 .../local/tasks/StreamsProcessorTask.java       |  22 ++--
 .../local/tasks/StreamsProviderTask.java        |  24 ++--
 .../apache/streams/local/tasks/StreamsTask.java |  13 +-
 .../jsonschema/LocalRuntimeConfiguration.json   |   8 +-
 .../streams/local/tasks/BasicTasksTest.java     |  60 +++------
 17 files changed, 286 insertions(+), 251 deletions(-)
----------------------------------------------------------------------



[3/7] incubator-streams git commit: Merge branch 'STREAMS-325' into STREAMS-226

Posted by sb...@apache.org.
Merge branch 'STREAMS-325' into STREAMS-226

* STREAMS-325:
  resolves STREAMS-325 #325

Conflicts:
	streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/40b9b1c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/40b9b1c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/40b9b1c5

Branch: refs/heads/master
Commit: 40b9b1c58796a68f0ccf14ff20915a0fac66f812
Parents: 24da4ca 8bd2255
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Sun May 24 19:09:18 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Sun May 24 19:09:18 2015 -0500

----------------------------------------------------------------------
 streams-monitoring/pom.xml                      | 11 +--
 .../tasks/BroadcastMonitorThread.java           | 83 ++++++++++++--------
 .../monitoring/MonitoringConfiguration.json     | 23 ++++++
 .../tasks/BroadcastMonitorThreadTest.java       | 24 ++++--
 .../local/builders/LocalStreamBuilder.java      | 19 ++++-
 5 files changed, 117 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/40b9b1c5/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --cc streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index aaf649c,801d2da..5d0377a
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@@ -209,4 -228,4 +228,4 @@@ public class BroadcastMonitorThread ext
      public void setDefaultWaitTime(long defaultWaitTime) {
          this.DEFAULT_WAIT_TIME = defaultWaitTime;
      }
--}
++}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/40b9b1c5/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --cc streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 5403130,f6d8f23..59d60df
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@@ -19,10 -19,12 +19,13 @@@
  package org.apache.streams.local.builders;
  
  import com.google.common.util.concurrent.Uninterruptibles;
+ import org.apache.streams.config.ComponentConfigurator;
+ import org.apache.streams.config.StreamsConfigurator;
  import org.apache.streams.core.*;
 +import org.apache.streams.local.LocalRuntimeConfiguration;
  import org.apache.streams.local.counters.StreamsTaskCounter;
  import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
+ import org.apache.streams.local.monitoring.MonitoringConfiguration;
  import org.apache.streams.local.queues.ThroughputQueue;
  import org.apache.streams.local.tasks.*;
  import org.apache.streams.monitoring.tasks.BroadcastMonitorThread;
@@@ -127,10 -120,24 +130,24 @@@ public class LocalStreamBuilder impleme
  
          setStreamIdentifier();
          if(this.streamConfig != null) {
 -            this.streamConfig.put(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
 +            this.streamConfig.setAdditionalProperty(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
          }
          this.useDeprecatedMonitors = false;
-         this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
+ 
+         /* for backward-compatibility with streamConfig */
+         MonitoringConfiguration monitoringConfiguration;
+         if( StreamsConfigurator.getConfig().hasPath("monitoring") ) {
+             monitoringConfiguration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration("monitoring");
+         } else {
+             monitoringConfiguration = new MonitoringConfiguration();
+         }
+         if( this.streamConfig != null &&
+             this.streamConfig.containsKey(BROADCAST_KEY)) {
+             monitoringConfiguration.setBroadcastURI(this.streamConfig.get(BROADCAST_KEY).toString());
+             if( this.streamConfig.containsKey(BROADCAST_INTERVAL_KEY))
+                 monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong(this.streamConfig.get(BROADCAST_INTERVAL_KEY).toString()));
+         }
+         this.broadcastMonitor = new BroadcastMonitorThread(monitoringConfiguration);
  
          this.futures = new HashMap<>();
      }


[2/7] incubator-streams git commit: initial refactoring

Posted by sb...@apache.org.
initial refactoring


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/24da4cab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/24da4cab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/24da4cab

Branch: refs/heads/master
Commit: 24da4cab0012fb2ae3ef04cd22971c978dc5106b
Parents: 6a05779
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Sun May 24 19:04:36 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Sun May 24 19:04:36 2015 -0500

----------------------------------------------------------------------
 .../main/jsonschema/StreamsConfiguration.json   | 38 +++++++++++++++-----
 .../tasks/BroadcastMonitorThread.java           |  2 +-
 .../local/builders/LocalStreamBuilder.java      | 27 +++++++-------
 .../streams/local/builders/StreamComponent.java | 13 +++----
 .../streams/local/tasks/BaseStreamsTask.java    | 23 +++---------
 .../streams/local/tasks/StreamsMergeTask.java   |  5 +--
 .../local/tasks/StreamsPersistWriterTask.java   |  9 ++---
 .../local/tasks/StreamsProcessorTask.java       | 11 +++---
 .../local/tasks/StreamsProviderTask.java        | 11 +++---
 .../apache/streams/local/tasks/StreamsTask.java |  3 +-
 .../jsonschema/LocalRuntimeConfiguration.json   |  8 +++--
 11 files changed, 85 insertions(+), 65 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-config/src/main/jsonschema/StreamsConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-config/src/main/jsonschema/StreamsConfiguration.json b/streams-config/src/main/jsonschema/StreamsConfiguration.json
index 2f68164..bfff501 100644
--- a/streams-config/src/main/jsonschema/StreamsConfiguration.json
+++ b/streams-config/src/main/jsonschema/StreamsConfiguration.json
@@ -8,20 +8,40 @@
     "javaType" : "org.apache.streams.config.StreamsConfiguration",
     "javaInterfaces": ["java.io.Serializable"],
     "properties": {
-        "pipeline": {
+        "batchSize": {
+            "type" : "integer",
+            "description" : "Default max batch size for batch-based persist readers and writers",
+            "default": 1000
+        },
+        "identifier": {
             "type" : "string",
+            "description" : "Name of streams pipeline",
             "default": "No-name Stream"
         },
-		"broadcastURI": {
-            "type": "string"
+        "parallelism": {
+            "type" : "integer",
+            "description" : "Default parallelism for processors",
+            "default": 1
+        },
+        "queueSize": {
+            "type" : "integer",
+            "description" : "Size of inter-component buffers",
+            "default": 1000
+        },
+        "batchFrequencyMs": {
+            "type": "integer",
+            "description": "Frequency of messaging between components",
+            "default": 5000
         },
-        "broadcastIntervalMs": {
-          "type": "integer"
+        "providerTimeoutMs": {
+            "type": "integer",
+            "description": "Kill the stream if all providers supply nothing for this long.",
+            "default": 1440000
         },
-        "timeoutMs": {
-          "type": "integer",
-          "default": "1440000",
-          "description": "Kill the stream if all providers supply nothing for this long."
+        "startedAt": {
+            "type": "integer",
+            "description": "Timestamp of beginning of stream execution",
+            "default": -1
         }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index 19b1067..aaf649c 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -52,7 +52,7 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
     private long DEFAULT_WAIT_TIME = 30000;
     private long waitTime;
     private ObjectMapper objectMapper;
-    private Map<String, Object> streamConfig;
+    private StreamsCon streamConfig;
     private URI broadcastURI = null;
     private MessagePersister messagePersister;
     private volatile boolean keepRunning;

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 601c6b7..5403130 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -20,6 +20,7 @@ package org.apache.streams.local.builders;
 
 import com.google.common.util.concurrent.Uninterruptibles;
 import org.apache.streams.core.*;
+import org.apache.streams.local.LocalRuntimeConfiguration;
 import org.apache.streams.local.counters.StreamsTaskCounter;
 import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
 import org.apache.streams.local.queues.ThroughputQueue;
@@ -54,7 +55,7 @@ public class LocalStreamBuilder implements StreamBuilder {
 
     private Map<String, StreamComponent> providers;
     private Map<String, StreamComponent> components;
-    private Map<String, Object> streamConfig;
+    private LocalRuntimeConfiguration streamConfig;
     private Map<StreamsTask, Future> futures;
     private ExecutorService executor;
     private ExecutorService monitor;
@@ -79,7 +80,9 @@ public class LocalStreamBuilder implements StreamBuilder {
     /**
      * Creates a local stream builder with a config object and default maximum internal queue size of 500
      * @param streamConfig
+     * @deprecated use LocalRuntimeConfiguration constructor instread
      */
+    @Deprecated
     public LocalStreamBuilder(Map<String, Object> streamConfig) {
         this(DEFAULT_QUEUE_SIZE, streamConfig);
     }
@@ -100,12 +103,19 @@ public class LocalStreamBuilder implements StreamBuilder {
      * @param streamConfig
      */
     public LocalStreamBuilder(int maxQueueCapacity, Map<String, Object> streamConfig) {
+        LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
+        for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
+            config.setAdditionalProperty(item.getKey(), item.getValue());
+        }
+        config.setBatchSize(new Long(maxQueueCapacity));
+    }
+
+    public LocalStreamBuilder(LocalRuntimeConfiguration streamConfig) {
         this.providers = new HashMap<String, StreamComponent>();
         this.components = new HashMap<String, StreamComponent>();
         this.streamConfig = streamConfig;
         this.totalTasks = 0;
         this.monitorTasks = 0;
-        this.maxQueueCapacity = maxQueueCapacity;
         final LocalStreamBuilder self = this;
         this.shutdownHook = new Thread() {
             @Override
@@ -117,7 +127,7 @@ public class LocalStreamBuilder implements StreamBuilder {
 
         setStreamIdentifier();
         if(this.streamConfig != null) {
-            this.streamConfig.put(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
+            this.streamConfig.setAdditionalProperty(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
         }
         this.useDeprecatedMonitors = false;
         this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
@@ -435,18 +445,11 @@ public class LocalStreamBuilder implements StreamBuilder {
 
     protected int getTimeout() {
         //Set the timeout of it is configured, otherwise signal downstream components to use their default
-        return streamConfig != null && streamConfig.containsKey(TIMEOUT_KEY) ? (Integer)streamConfig.get(TIMEOUT_KEY) : -1;
+        return streamConfig.getProviderTimeoutMs().intValue();
     }
 
     private void setStreamIdentifier() {
-        if(streamConfig != null &&
-                streamConfig.containsKey(STREAM_IDENTIFIER_KEY) &&
-                streamConfig.get(STREAM_IDENTIFIER_KEY) != null &&
-                streamConfig.get(STREAM_IDENTIFIER_KEY).toString().length() > 0) {
-            this.streamIdentifier = streamConfig.get(STREAM_IDENTIFIER_KEY).toString();
-        } else {
-            this.streamIdentifier = DEFAULT_STREAM_IDENTIFIER;
-        }
+        this.streamIdentifier = streamConfig.getIdentifier();
     }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
index 31c5981..4f2a507 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
@@ -19,6 +19,7 @@
 package org.apache.streams.local.builders;
 
 import com.google.common.collect.Lists;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.*;
 import org.apache.streams.local.tasks.StreamsPersistWriterTask;
 import org.apache.streams.local.tasks.StreamsProcessorTask;
@@ -54,14 +55,14 @@ public class StreamComponent {
 
     private List<StreamsTask> tasks;
 
-    private Map<String, Object> streamConfig;
+    private StreamsConfiguration streamConfig;
 
     /**
      *
      * @param id
      * @param provider
      */
-    public StreamComponent(String id, StreamsProvider provider, boolean perpetual, Map<String, Object> streamConfig) {
+    public StreamComponent(String id, StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
         this.id = id;
         this.provider = provider;
         this.perpetual = perpetual;
@@ -76,7 +77,7 @@ public class StreamComponent {
      * @param start
      * @param end
      */
-    public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end, Map<String, Object> streamConfig) {
+    public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end, StreamsConfiguration streamConfig) {
         this.id = id;
         this.provider = provider;
         this.dateRange = new DateTime[2];
@@ -93,7 +94,7 @@ public class StreamComponent {
      * @param provider
      * @param sequence
      */
-    public StreamComponent(String id, StreamsProvider provider, BigInteger sequence, Map<String, Object> streamConfig) {
+    public StreamComponent(String id, StreamsProvider provider, BigInteger sequence, StreamsConfiguration streamConfig) {
         this.id = id;
         this.provider = provider;
         this.sequence = sequence;
@@ -107,7 +108,7 @@ public class StreamComponent {
      * @param inQueue
      * @param numTasks
      */
-    public StreamComponent(String id, StreamsProcessor processor, BlockingQueue<StreamsDatum> inQueue, int numTasks, Map<String, Object> streamConfig) {
+    public StreamComponent(String id, StreamsProcessor processor, BlockingQueue<StreamsDatum> inQueue, int numTasks, StreamsConfiguration streamConfig) {
         this.id = id;
         this.processor = processor;
         this.inQueue = inQueue;
@@ -123,7 +124,7 @@ public class StreamComponent {
      * @param inQueue
      * @param numTasks
      */
-    public StreamComponent(String id, StreamsPersistWriter writer, BlockingQueue<StreamsDatum> inQueue, int numTasks, Map<String, Object> streamConfig) {
+    public StreamComponent(String id, StreamsPersistWriter writer, BlockingQueue<StreamsDatum> inQueue, int numTasks, StreamsConfiguration streamConfig) {
         this.id = id;
         this.writer = writer;
         this.inQueue = inQueue;

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
index c0c7ea6..3daa110 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
@@ -21,6 +21,7 @@ package org.apache.streams.local.tasks;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.StreamsDatum;
 import org.apache.streams.jackson.StreamsJacksonMapper;
 import org.apache.streams.local.builders.LocalStreamBuilder;
@@ -46,12 +47,12 @@ public abstract class BaseStreamsTask implements StreamsTask {
     private List<BlockingQueue<StreamsDatum>> outQueues = new LinkedList<BlockingQueue<StreamsDatum>>();
     private int inIndex = 0;
     private ObjectMapper mapper;
-    protected Map<String, Object> streamConfig;
+    protected StreamsConfiguration streamConfig;
 
     private long startedAt;
     private String streamIdentifier;
 
-    public BaseStreamsTask(Map<String, Object> config) {
+    public BaseStreamsTask(StreamsConfiguration config) {
         this.mapper = StreamsJacksonMapper.getInstance();
         this.mapper.registerSubtypes(Activity.class);
         this.streamConfig = config;
@@ -205,14 +206,7 @@ public abstract class BaseStreamsTask implements StreamsTask {
     }
 
     public void setStartedAt() {
-        if(streamConfig != null &&
-                streamConfig.containsKey(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY) &&
-                streamConfig.get(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY) != null &&
-                streamConfig.get(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY) instanceof Long) {
-            this.startedAt = Long.parseLong(streamConfig.get(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY).toString());
-        } else {
-            this.startedAt = -1;
-        }
+        this.startedAt = streamConfig.getStartedAt();
     }
 
     public String getStreamIdentifier() {
@@ -220,14 +214,7 @@ public abstract class BaseStreamsTask implements StreamsTask {
     }
 
     public void setStreamIdentifier() {
-        if(streamConfig != null &&
-                streamConfig.containsKey(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) &&
-                streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) != null &&
-                streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY).toString().length() > 0) {
-            this.streamIdentifier = streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY).toString();
-        } else {
-            this.streamIdentifier = LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER;
-        }
+        this.streamIdentifier = streamConfig.getIdentifier();
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
index ed38fe8..49cb074 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -18,6 +18,7 @@
 
 package org.apache.streams.local.tasks;
 
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.StreamsDatum;
 import org.apache.streams.local.counters.StreamsTaskCounter;
 import org.apache.commons.lang.NotImplementedException;
@@ -44,7 +45,7 @@ public class StreamsMergeTask extends BaseStreamsTask {
         this(sleepTime, null);
     }
 
-    public StreamsMergeTask(long sleepTime, Map<String, Object> streamConfig) {
+    public StreamsMergeTask(long sleepTime, StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.sleepTime = sleepTime;
         this.keepRunning = new AtomicBoolean(true);
@@ -56,7 +57,7 @@ public class StreamsMergeTask extends BaseStreamsTask {
     }
 
     @Override
-    public void setStreamConfig(Map<String, Object> config) {
+    public void setStreamConfig(StreamsConfiguration config) {
 
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index 860a8f0..d6bf2ec 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -19,6 +19,7 @@
 package org.apache.streams.local.tasks;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.*;
 import org.apache.streams.core.util.DatumUtils;
 import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -40,7 +41,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
     private StreamsPersistWriter writer;
     private long sleepTime = DEFAULT_SLEEP_TIME_MS * 10;
     private AtomicBoolean keepRunning;
-    private Map<String, Object> streamConfig;
+    private StreamsConfiguration streamConfig;
     private BlockingQueue<StreamsDatum> inQueue;
     private AtomicBoolean isRunning;
     private AtomicBoolean blocked;
@@ -62,7 +63,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
         this(writer, DEFAULT_SLEEP_TIME_MS, null);
     }
 
-    public StreamsPersistWriterTask(StreamsPersistWriter writer, Map<String, Object> streamConfig) {
+    public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
         this(writer, DEFAULT_SLEEP_TIME_MS, streamConfig);
     }
 
@@ -71,7 +72,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
      * @param writer writer to execute in task
      * @param sleepTime time to sleep when inbound queue is empty.
      */
-    public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime, Map<String, Object> streamConfig) {
+    public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime, StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.writer = writer;
         this.sleepTime = sleepTime;
@@ -86,7 +87,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
     }
 
     @Override
-    public void setStreamConfig(Map<String, Object> config) {
+    public void setStreamConfig(StreamsConfiguration config) {
         this.streamConfig = config;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index d03dfa3..5e04b2f 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -19,6 +19,7 @@
 package org.apache.streams.local.tasks;
 
 import com.google.common.collect.Maps;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.*;
 import org.apache.streams.core.util.DatumUtils;
 import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -41,7 +42,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
     private StreamsProcessor processor;
     private long sleepTime;
     private AtomicBoolean keepRunning;
-    private Map<String, Object> streamConfig;
+    private StreamsConfiguration streamConfig;
     private BlockingQueue<StreamsDatum> inQueue;
     private AtomicBoolean isRunning;
     private AtomicBoolean blocked;
@@ -67,14 +68,14 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
      * @param processor
      * @param streamConfig
      */
-    public StreamsProcessorTask(StreamsProcessor processor, Map<String, Object> streamConfig) { this(processor, DEFAULT_SLEEP_TIME_MS, streamConfig); }
+    public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) { this(processor, DEFAULT_SLEEP_TIME_MS, streamConfig); }
 
     /**
      *
      * @param processor processor to run in task
      * @param sleepTime time to sleep when incoming queue is empty
      */
-    public StreamsProcessorTask(StreamsProcessor processor, long sleepTime, Map<String, Object> streamConfig) {
+    public StreamsProcessorTask(StreamsProcessor processor, long sleepTime, StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.processor = processor;
         this.sleepTime = sleepTime;
@@ -94,7 +95,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
     }
 
     @Override
-    public void setStreamConfig(Map<String, Object> config) {
+    public void setStreamConfig(StreamsConfiguration config) {
         this.streamConfig = config;
     }
 
@@ -119,7 +120,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
                 StreamsDatum datum = null;
                 try {
                     this.blocked.set(true);
-                    datum = this.inQueue.poll(5, TimeUnit.SECONDS);
+                    datum = this.inQueue.poll(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
                 } catch (InterruptedException ie) {
                     LOGGER.debug("Received InteruptedException, shutting down and re-applying interrupt status.");
                     this.keepRunning.set(false);

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index c446a72..cb36a24 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -19,6 +19,7 @@
 package org.apache.streams.local.tasks;
 
 import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.*;
 import org.apache.streams.core.util.DatumUtils;
 import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -62,7 +63,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
     private Type type;
     private BigInteger sequence;
     private DateTime[] dateRange;
-    private Map<String, Object> config;
+    private StreamsConfiguration config;
 
     private int timeout;
     private long sleepTime;
@@ -74,7 +75,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
      * Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
      * @param provider
      */
-    public StreamsProviderTask(StreamsProvider provider, boolean perpetual, Map<String, Object> streamConfig) {
+    public StreamsProviderTask(StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.provider = provider;
         if( perpetual )
@@ -90,7 +91,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
      * @param provider
      * @param sequence
      */
-    public StreamsProviderTask(StreamsProvider provider, BigInteger sequence, Map<String, Object> streamConfig) {
+    public StreamsProviderTask(StreamsProvider provider, BigInteger sequence, StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.provider = provider;
         this.type = Type.READ_NEW;
@@ -105,7 +106,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
      * @param start
      * @param end
      */
-    public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end, Map<String, Object> streamConfig) {
+    public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end, StreamsConfiguration streamConfig) {
         super(streamConfig);
         this.provider = provider;
         this.type = Type.READ_RANGE;
@@ -141,7 +142,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
     }
 
     @Override
-    public void setStreamConfig(Map<String, Object> config) {
+    public void setStreamConfig(StreamsConfiguration config) {
         this.config = config;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
index 8423095..e98dc0b 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -18,6 +18,7 @@
 
 package org.apache.streams.local.tasks;
 
+import org.apache.streams.config.StreamsConfiguration;
 import org.apache.streams.core.StreamsDatum;
 import org.apache.streams.local.counters.StreamsTaskCounter;
 
@@ -68,7 +69,7 @@ public interface StreamsTask extends Runnable{
      * Set the configuration object that will shared and passed to all instances of StreamsTask.
      * @param config optional configuration information
      */
-    public void setStreamConfig(Map<String, Object> config);
+    public void setStreamConfig(StreamsConfiguration config);
 
     /**
      * Returns true when the task has not completed. Returns false otherwise

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
index c7f79a0..36177d9 100644
--- a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
+++ b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
@@ -6,10 +6,14 @@
     "id": "#",
     "type": "object",
     "javaType" : "org.apache.streams.local.LocalRuntimeConfiguration",
+    "extends": {
+        "$ref": "../../../../../streams-config/src/main/jsonschema/StreamsConfiguration.json"
+    },
     "javaInterfaces": ["java.io.Serializable"],
     "properties": {
-        "providerTimeoutMs": {
-            "type" : "integer"
+        "monitoring": {
+            "type" : "object",
+            "javaType" : "org.apache.streams.local.monitoring.MonitoringConfiguration"
         }
     }
 }
\ No newline at end of file