You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/06/20 00:02:39 UTC
[1/7] incubator-streams git commit: resolves STREAMS-325 #325
Repository: incubator-streams
Updated Branches:
refs/heads/master e7bf641d3 -> 39c07f1d6
resolves STREAMS-325 #325
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/8bd22553
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/8bd22553
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/8bd22553
Branch: refs/heads/master
Commit: 8bd225537944305e84a27547b05c24c6c459e7bf
Parents: 6a05779
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Tue May 19 18:00:15 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Tue May 19 18:00:15 2015 -0500
----------------------------------------------------------------------
streams-monitoring/pom.xml | 11 +--
.../tasks/BroadcastMonitorThread.java | 81 ++++++++++++--------
.../monitoring/MonitoringConfiguration.json | 23 ++++++
.../tasks/BroadcastMonitorThreadTest.java | 24 ++++--
.../local/builders/LocalStreamBuilder.java | 19 ++++-
5 files changed, 116 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/pom.xml
----------------------------------------------------------------------
diff --git a/streams-monitoring/pom.xml b/streams-monitoring/pom.xml
index 2f66a07..0e3b86e 100644
--- a/streams-monitoring/pom.xml
+++ b/streams-monitoring/pom.xml
@@ -42,6 +42,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-config</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcomponents-core</artifactId>
<type>pom</type>
@@ -88,11 +93,7 @@
<addCompileSourceRoot>true</addCompileSourceRoot>
<generateBuilders>true</generateBuilders>
<sourcePaths>
- <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/Broadcast.json</sourcePath>
- <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/StreamsTaskCounterBroadcast.json</sourcePath>
- <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/ThroughputQueueBroadcast.json</sourcePath>
- <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/DatumStatusCounterBroadcast.json</sourcePath>
- <sourcePath>src/main/jsonschema/org/apache/streams/pojo/json/MemoryUsageBroadcast.json</sourcePath>
+ <sourcePath>src/main/jsonschema</sourcePath>
</sourcePaths>
<outputDirectory>target/generated-sources/jsonschema2pojo</outputDirectory>
<targetPackage>org.apache.streams.pojo.json</targetPackage>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index 19b1067..801d2da 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -21,7 +21,10 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.collect.Lists;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.jackson.*;
+import org.apache.streams.local.monitoring.MonitoringConfiguration;
import org.apache.streams.monitoring.persist.MessagePersister;
import org.apache.streams.monitoring.persist.impl.BroadcastMessagePersister;
import org.apache.streams.monitoring.persist.impl.LogstashUdpMessagePersister;
@@ -52,38 +55,33 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
private long DEFAULT_WAIT_TIME = 30000;
private long waitTime;
private ObjectMapper objectMapper;
- private Map<String, Object> streamConfig;
+ private MonitoringConfiguration configuration;
private URI broadcastURI = null;
private MessagePersister messagePersister;
private volatile boolean keepRunning;
+ /**
+ * DEPRECATED
+ * Please initialize logging with monitoring object via typesafe
+ * @param streamConfig
+ */
+ @Deprecated
public BroadcastMonitorThread(Map<String, Object> streamConfig) {
- keepRunning = true;
- this.streamConfig = streamConfig;
+ this(new ObjectMapper().convertValue(streamConfig, MonitoringConfiguration.class));
+ }
- LOGGER.info("BroadcastMonitorThread starting" + streamConfig);
+ public BroadcastMonitorThread(MonitoringConfiguration configuration) {
- server = ManagementFactory.getPlatformMBeanServer();
+ this.configuration = configuration;
- setBroadcastURI();
- setWaitTime();
+ LOGGER.info("BroadcastMonitorThread created");
- if( broadcastURI != null ) {
- if (broadcastURI.getScheme().equals("http")) {
- messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
- } else if (broadcastURI.getScheme().equals("udp")) {
- messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
- } else {
- LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
- throw new RuntimeException();
- }
- } else {
- messagePersister = new SLF4JMessagePersister();
- }
+ prepare();
initializeObjectMapper();
- LOGGER.info("BroadcastMonitorThread started");
+ LOGGER.info("BroadcastMonitorThread initialized");
+
}
/**
@@ -153,12 +151,10 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
* Go through streams config and set the broadcastURI (if present)
*/
private void setBroadcastURI() {
- if(streamConfig != null &&
- streamConfig.containsKey("broadcastURI") &&
- streamConfig.get("broadcastURI") != null &&
- streamConfig.get("broadcastURI") instanceof String) {
+ if(configuration != null &&
+ configuration.getBroadcastURI() != null) {
try {
- broadcastURI = new URI(streamConfig.get("broadcastURI").toString());
+ broadcastURI = new URI(configuration.getBroadcastURI());
} catch (URISyntaxException e) {
e.printStackTrace();
}
@@ -170,12 +166,9 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
*/
private void setWaitTime() {
try {
- if (streamConfig != null &&
- streamConfig.containsKey("monitoring_broadcast_interval_ms") &&
- streamConfig.get("monitoring_broadcast_interval_ms") != null &&
- (streamConfig.get("monitoring_broadcast_interval_ms") instanceof Long ||
- streamConfig.get("monitoring_broadcast_interval_ms") instanceof Integer)) {
- waitTime = Long.parseLong(streamConfig.get("monitoring_broadcast_interval_ms").toString());
+ if (configuration != null &&
+ configuration.getMonitoringBroadcastIntervalMs() != null ) {
+ waitTime = configuration.getMonitoringBroadcastIntervalMs();
} else {
waitTime = DEFAULT_WAIT_TIME;
}
@@ -189,6 +182,32 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
}
}
+ public void prepare() {
+
+ keepRunning = true;
+
+ LOGGER.info("BroadcastMonitorThread setup " + configuration);
+
+ server = ManagementFactory.getPlatformMBeanServer();
+
+ setBroadcastURI();
+ setWaitTime();
+
+ if( broadcastURI != null ) {
+ if (broadcastURI.getScheme().equals("http")) {
+ messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
+ } else if (broadcastURI.getScheme().equals("udp")) {
+ messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
+ } else {
+ LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
+ throw new RuntimeException();
+ }
+ } else {
+ messagePersister = new SLF4JMessagePersister();
+ }
+
+ }
+
public void shutdown() {
this.keepRunning = false;
LOGGER.debug("Shutting down BroadcastMonitor Thread");
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json b/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
new file mode 100644
index 0000000..2c50cef
--- /dev/null
+++ b/streams-monitoring/src/main/jsonschema/org/apache/streams/local/monitoring/MonitoringConfiguration.json
@@ -0,0 +1,23 @@
+{
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "$license": [
+ "http://www.apache.org/licenses/LICENSE-2.0"
+ ],
+ "id": "#",
+ "type": "object",
+ "title": "object",
+ "javaType": "org.apache.streams.local.monitoring.MonitoringConfiguration",
+ "javaInterfaces": ["java.io.Serializable"],
+ "description": "Configuration for local runtime monitoring",
+ "properties": {
+ "broadcastURI": {
+ "type": "string",
+ "description": "URI for monitoring statistics"
+ },
+ "monitoring_broadcast_interval_ms": {
+ "type": "integer",
+ "description": "Milliseconds between publish events",
+ "default": 30000
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
index 901888e..7ad315f 100644
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
@@ -18,6 +18,8 @@
package org.apache.streams.monitoring.tasks;
import com.google.common.collect.Maps;
+import org.apache.streams.local.monitoring.MonitoringConfiguration;
+import org.junit.Ignore;
import org.junit.Test;
import java.util.Map;
@@ -28,17 +30,29 @@ public class BroadcastMonitorThreadTest {
private ExecutorService executor;
@Test
- public void testThreadNullConfig() {
- BroadcastMonitorThread thread = new BroadcastMonitorThread(null);
+ public void testThreadEmptyBeanConfig() {
+ MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
+ BroadcastMonitorThread thread = new BroadcastMonitorThread(monitoringConfiguration);
+ testThread(thread);
}
@Test
- public void testThread() {
+ public void testThreadEmptyMapConfig() {
+ Map<String, Object> map = Maps.newHashMap();
+ BroadcastMonitorThread thread = new BroadcastMonitorThread(map);
+ testThread(thread);
+ }
+
+ @Test
+ public void testThreadFakeMapConfig() {
Map<String, Object> config = Maps.newHashMap();
config.put("broadcastURI", "http://fakeurl.com/fake");
-
BroadcastMonitorThread thread = new BroadcastMonitorThread(config);
- thread.setDefaultWaitTime(30000L);
+ testThread(thread);
+ }
+
+ public void testThread(BroadcastMonitorThread thread) {
+ thread.setDefaultWaitTime(3000L);
long testRunLength = thread.getDefaultWaitTime() * 1;
executor = Executors.newFixedThreadPool(1);
executor.submit(thread);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/8bd22553/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 601c6b7..f6d8f23 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -19,9 +19,12 @@
package org.apache.streams.local.builders;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.*;
import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
+import org.apache.streams.local.monitoring.MonitoringConfiguration;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.tasks.*;
import org.apache.streams.monitoring.tasks.BroadcastMonitorThread;
@@ -120,7 +123,21 @@ public class LocalStreamBuilder implements StreamBuilder {
this.streamConfig.put(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
}
this.useDeprecatedMonitors = false;
- this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
+
+ /* for backward-compatibility with streamConfig */
+ MonitoringConfiguration monitoringConfiguration;
+ if( StreamsConfigurator.getConfig().hasPath("monitoring") ) {
+ monitoringConfiguration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration("monitoring");
+ } else {
+ monitoringConfiguration = new MonitoringConfiguration();
+ }
+ if( this.streamConfig != null &&
+ this.streamConfig.containsKey(BROADCAST_KEY)) {
+ monitoringConfiguration.setBroadcastURI(this.streamConfig.get(BROADCAST_KEY).toString());
+ if( this.streamConfig.containsKey(BROADCAST_INTERVAL_KEY))
+ monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong(this.streamConfig.get(BROADCAST_INTERVAL_KEY).toString()));
+ }
+ this.broadcastMonitor = new BroadcastMonitorThread(monitoringConfiguration);
this.futures = new HashMap<>();
}
[4/7] incubator-streams git commit: completed,
all tests passing resolves #226 resolves #294 resolves #325
Posted by sb...@apache.org.
completed, all tests passing
resolves #226
resolves #294
resolves #325
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/4081987c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/4081987c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/4081987c
Branch: refs/heads/master
Commit: 4081987c3636886457f0dd7eff731e88c8954471
Parents: 40b9b1c
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Mon May 25 18:51:10 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Mon May 25 18:51:10 2015 -0500
----------------------------------------------------------------------
.../main/jsonschema/StreamsConfiguration.json | 20 +++---
.../tasks/BroadcastMonitorThread.java | 11 ++-
.../tasks/BroadcastMonitorThreadTest.java | 5 +-
.../local/builders/LocalStreamBuilder.java | 75 +++++++++++---------
.../streams/local/tasks/BaseStreamsTask.java | 22 +++---
.../local/tasks/StreamsPersistWriterTask.java | 17 ++---
.../local/tasks/StreamsProcessorTask.java | 15 +---
.../local/tasks/StreamsProviderTask.java | 13 ++--
.../apache/streams/local/tasks/StreamsTask.java | 10 ---
.../streams/local/tasks/BasicTasksTest.java | 60 +++++-----------
10 files changed, 107 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-config/src/main/jsonschema/StreamsConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-config/src/main/jsonschema/StreamsConfiguration.json b/streams-config/src/main/jsonschema/StreamsConfiguration.json
index bfff501..d331df2 100644
--- a/streams-config/src/main/jsonschema/StreamsConfiguration.json
+++ b/streams-config/src/main/jsonschema/StreamsConfiguration.json
@@ -8,6 +8,11 @@
"javaType" : "org.apache.streams.config.StreamsConfiguration",
"javaInterfaces": ["java.io.Serializable"],
"properties": {
+ "batchFrequencyMs": {
+ "type": "integer",
+ "description": "Frequency of messaging between components",
+ "default": 5000
+ },
"batchSize": {
"type" : "integer",
"description" : "Default max batch size for batch-based persist readers and writers",
@@ -23,21 +28,16 @@
"description" : "Default parallelism for processors",
"default": 1
},
- "queueSize": {
- "type" : "integer",
- "description" : "Size of inter-component buffers",
- "default": 1000
- },
- "batchFrequencyMs": {
- "type": "integer",
- "description": "Frequency of messaging between components",
- "default": 5000
- },
"providerTimeoutMs": {
"type": "integer",
"description": "Kill the stream if all providers supply nothing for this long.",
"default": 1440000
},
+ "queueSize": {
+ "type" : "integer",
+ "description" : "Size of inter-component buffers",
+ "default": 1000
+ },
"startedAt": {
"type": "integer",
"description": "Timestamp of beginning of stream execution",
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index 5d0377a..b9db630 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.collect.Lists;
import org.apache.streams.config.ComponentConfigurator;
import org.apache.streams.config.StreamsConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.jackson.*;
import org.apache.streams.local.monitoring.MonitoringConfiguration;
import org.apache.streams.monitoring.persist.MessagePersister;
@@ -55,6 +56,7 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
private long DEFAULT_WAIT_TIME = 30000;
private long waitTime;
private ObjectMapper objectMapper;
+ private StreamsConfiguration streamConfig;
private MonitoringConfiguration configuration;
private URI broadcastURI = null;
private MessagePersister messagePersister;
@@ -67,11 +69,16 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
*/
@Deprecated
public BroadcastMonitorThread(Map<String, Object> streamConfig) {
- this(new ObjectMapper().convertValue(streamConfig, MonitoringConfiguration.class));
+ this(new ObjectMapper().convertValue(streamConfig, StreamsConfiguration.class));
}
- public BroadcastMonitorThread(MonitoringConfiguration configuration) {
+ public BroadcastMonitorThread(StreamsConfiguration streamConfig) {
+ this(streamConfig, new ObjectMapper().convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
+ }
+
+ public BroadcastMonitorThread(StreamsConfiguration streamConfig, MonitoringConfiguration configuration) {
+ this.streamConfig = streamConfig;
this.configuration = configuration;
LOGGER.info("BroadcastMonitorThread created");
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
index 7ad315f..427757b 100644
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
@@ -18,6 +18,7 @@
package org.apache.streams.monitoring.tasks;
import com.google.common.collect.Maps;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.local.monitoring.MonitoringConfiguration;
import org.junit.Ignore;
import org.junit.Test;
@@ -31,8 +32,8 @@ public class BroadcastMonitorThreadTest {
@Test
public void testThreadEmptyBeanConfig() {
- MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
- BroadcastMonitorThread thread = new BroadcastMonitorThread(monitoringConfiguration);
+ StreamsConfiguration streamsConfiguration = new StreamsConfiguration();
+ BroadcastMonitorThread thread = new BroadcastMonitorThread(streamsConfiguration);
testThread(thread);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 59d60df..4175e10 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -18,8 +18,10 @@
package org.apache.streams.local.builders;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.*;
import org.apache.streams.local.LocalRuntimeConfiguration;
@@ -74,16 +76,16 @@ public class LocalStreamBuilder implements StreamBuilder {
private boolean useDeprecatedMonitors;
/**
- * Creates a local stream builder with no config object and default maximum internal queue size of 500
+ * Creates a local stream builder with all configuration resolved by typesafe
*/
- public LocalStreamBuilder(){
- this(DEFAULT_QUEUE_SIZE, null);
+ public LocalStreamBuilder() {
+ this(new ObjectMapper().convertValue(StreamsConfigurator.detectConfiguration(), LocalRuntimeConfiguration.class));
}
/**
* Creates a local stream builder with a config object and default maximum internal queue size of 500
* @param streamConfig
- * @deprecated use LocalRuntimeConfiguration constructor instread
+ * @deprecated use LocalRuntimeConfiguration constructor instead
*/
@Deprecated
public LocalStreamBuilder(Map<String, Object> streamConfig) {
@@ -94,7 +96,10 @@ public class LocalStreamBuilder implements StreamBuilder {
* Creates a local stream builder with no config object. If maxQueueCapacity is less than 1 the queue is
* unbounded.
* @param maxQueueCapacity
+ *
+ * @deprecated use LocalRuntimeConfiguration constructor instead
*/
+ @Deprecated
public LocalStreamBuilder(int maxQueueCapacity) {
this(maxQueueCapacity, null);
}
@@ -102,23 +107,41 @@ public class LocalStreamBuilder implements StreamBuilder {
/**
* Creates a local stream builder with a config object. If maxQueueCapacity is less than 1 the queue is
* unbounded.
+ *
* @param maxQueueCapacity
* @param streamConfig
+ *
+ * @deprecated use LocalRuntimeConfiguration constructor instead
*/
+ @Deprecated
public LocalStreamBuilder(int maxQueueCapacity, Map<String, Object> streamConfig) {
- LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
- for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
- config.setAdditionalProperty(item.getKey(), item.getValue());
+ this(new LocalRuntimeConfiguration());
+ this.streamConfig.setQueueSize(new Long(maxQueueCapacity));
+ if( streamConfig != null && streamConfig.get(LocalStreamBuilder.TIMEOUT_KEY) != null )
+ this.streamConfig.setProviderTimeoutMs(new Long((Integer) (streamConfig.get(LocalStreamBuilder.TIMEOUT_KEY))));
+ if( streamConfig != null && streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) != null )
+ this.streamConfig.setIdentifier((String)streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY));
+ if( streamConfig != null && streamConfig.get(LocalStreamBuilder.BROADCAST_KEY) != null ) {
+ MonitoringConfiguration monitoringConfiguration = new MonitoringConfiguration();
+ monitoringConfiguration.setBroadcastURI((String)streamConfig.get(LocalStreamBuilder.BROADCAST_KEY));
+ if(streamConfig.get(LocalStreamBuilder.BROADCAST_INTERVAL_KEY) != null)
+ monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong((String)streamConfig.get(LocalStreamBuilder.BROADCAST_INTERVAL_KEY)));
+ this.streamConfig.setMonitoring(monitoringConfiguration);
}
- config.setBatchSize(new Long(maxQueueCapacity));
}
public LocalStreamBuilder(LocalRuntimeConfiguration streamConfig) {
+ this.streamConfig = streamConfig;
this.providers = new HashMap<String, StreamComponent>();
this.components = new HashMap<String, StreamComponent>();
- this.streamConfig = streamConfig;
this.totalTasks = 0;
this.monitorTasks = 0;
+ this.futures = new HashMap<>();
+ }
+
+ public void prepare() {
+ this.streamIdentifier = streamConfig.getIdentifier();
+ this.streamConfig.setStartedAt(startedAt.getMillis());
final LocalStreamBuilder self = this;
this.shutdownHook = new Thread() {
@Override
@@ -127,29 +150,8 @@ public class LocalStreamBuilder implements StreamBuilder {
self.stopInternal(true);
}
};
-
- setStreamIdentifier();
- if(this.streamConfig != null) {
- this.streamConfig.setAdditionalProperty(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
- }
this.useDeprecatedMonitors = false;
-
- /* for backward-compatibility with streamConfig */
- MonitoringConfiguration monitoringConfiguration;
- if( StreamsConfigurator.getConfig().hasPath("monitoring") ) {
- monitoringConfiguration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration("monitoring");
- } else {
- monitoringConfiguration = new MonitoringConfiguration();
- }
- if( this.streamConfig != null &&
- this.streamConfig.containsKey(BROADCAST_KEY)) {
- monitoringConfiguration.setBroadcastURI(this.streamConfig.get(BROADCAST_KEY).toString());
- if( this.streamConfig.containsKey(BROADCAST_INTERVAL_KEY))
- monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong(this.streamConfig.get(BROADCAST_INTERVAL_KEY).toString()));
- }
- this.broadcastMonitor = new BroadcastMonitorThread(monitoringConfiguration);
-
- this.futures = new HashMap<>();
+ this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
}
public void setUseDeprecatedMonitors(boolean useDeprecatedMonitors) {
@@ -225,6 +227,7 @@ public class LocalStreamBuilder implements StreamBuilder {
*/
@Override
public void start() {
+ prepare();
attachShutdownHandler();
boolean isRunning = true;
this.executor = new ShutdownStreamOnUnhandleThrowableThreadPoolExecutor(this.totalTasks, this);
@@ -465,8 +468,14 @@ public class LocalStreamBuilder implements StreamBuilder {
return streamConfig.getProviderTimeoutMs().intValue();
}
- private void setStreamIdentifier() {
- this.streamIdentifier = streamConfig.getIdentifier();
+ private LocalRuntimeConfiguration convertConfiguration(Map<String, Object> streamConfig) {
+ LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
+ if( streamConfig != null ) {
+ for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
+ config.setAdditionalProperty(item.getKey(), item.getValue());
+ }
+ }
+ return config;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
index 3daa110..8bc7769 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
@@ -22,12 +22,14 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.local.builders.LocalStreamBuilder;
import org.apache.streams.pojo.json.Activity;
import org.apache.streams.util.ComponentUtils;
import org.apache.streams.util.SerializationUtil;
+import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,15 +51,14 @@ public abstract class BaseStreamsTask implements StreamsTask {
private ObjectMapper mapper;
protected StreamsConfiguration streamConfig;
- private long startedAt;
- private String streamIdentifier;
-
public BaseStreamsTask(StreamsConfiguration config) {
this.mapper = StreamsJacksonMapper.getInstance();
this.mapper.registerSubtypes(Activity.class);
- this.streamConfig = config;
+ if( config != null )
+ this.streamConfig = config;
+ else
+ this.streamConfig = StreamsConfigurator.detectConfiguration();
- setStreamIdentifier();
setStartedAt();
}
@@ -202,20 +203,15 @@ public abstract class BaseStreamsTask implements StreamsTask {
}
public long getStartedAt() {
- return startedAt;
+ return streamConfig.getStartedAt();
}
public void setStartedAt() {
- this.startedAt = streamConfig.getStartedAt();
+ streamConfig.setStartedAt(DateTime.now().getMillis());
}
public String getStreamIdentifier() {
- return streamIdentifier;
- }
-
- public void setStreamIdentifier() {
- this.streamIdentifier = streamConfig.getIdentifier();
+ return streamConfig.getIdentifier();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index d6bf2ec..a529c69 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -39,7 +39,6 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
private final static Logger LOGGER = LoggerFactory.getLogger(StreamsPersistWriterTask.class);
private StreamsPersistWriter writer;
- private long sleepTime = DEFAULT_SLEEP_TIME_MS * 10;
private AtomicBoolean keepRunning;
private StreamsConfiguration streamConfig;
private BlockingQueue<StreamsDatum> inQueue;
@@ -60,22 +59,18 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
* @param writer writer to execute in task
*/
public StreamsPersistWriterTask(StreamsPersistWriter writer) {
- this(writer, DEFAULT_SLEEP_TIME_MS, null);
- }
-
- public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
- this(writer, DEFAULT_SLEEP_TIME_MS, streamConfig);
+ this(writer, null);
}
/**
*
* @param writer writer to execute in task
- * @param sleepTime time to sleep when inbound queue is empty.
+ * @param streamConfig stream config
*/
- public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime, StreamsConfiguration streamConfig) {
+ public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
super(streamConfig);
+ this.streamConfig = super.streamConfig;
this.writer = writer;
- this.sleepTime = sleepTime;
this.keepRunning = new AtomicBoolean(true);
this.isRunning = new AtomicBoolean(true);
this.blocked = new AtomicBoolean(false);
@@ -141,11 +136,11 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
LOGGER.debug("Received null StreamsDatum @ writer : {}", this.writer.getClass().getName());
}
}
- Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
} catch(Exception e) {
LOGGER.error("Failed to execute Persist Writer {}",this.writer.getClass().getSimpleName(), e);
} finally {
- Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS);
+ Uninterruptibles.sleepUninterruptibly(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
this.writer.cleanUp();
this.isRunning.set(false);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index 5e04b2f..72b3eb6 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -40,7 +40,6 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
private StreamsProcessor processor;
- private long sleepTime;
private AtomicBoolean keepRunning;
private StreamsConfiguration streamConfig;
private BlockingQueue<StreamsDatum> inQueue;
@@ -60,25 +59,17 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
* @param processor process to run in task
*/
public StreamsProcessorTask(StreamsProcessor processor) {
- this(processor, DEFAULT_SLEEP_TIME_MS, null);
+ this(processor, new StreamsConfiguration());
}
/**
- *
* @param processor
* @param streamConfig
*/
- public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) { this(processor, DEFAULT_SLEEP_TIME_MS, streamConfig); }
-
- /**
- *
- * @param processor processor to run in task
- * @param sleepTime time to sleep when incoming queue is empty
- */
- public StreamsProcessorTask(StreamsProcessor processor, long sleepTime, StreamsConfiguration streamConfig) {
+ public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) {
super(streamConfig);
+ this.streamConfig = super.streamConfig;
this.processor = processor;
- this.sleepTime = sleepTime;
this.keepRunning = new AtomicBoolean(true);
this.isRunning = new AtomicBoolean(true);
this.blocked = new AtomicBoolean(true);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index cb36a24..53fe7c6 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -77,13 +77,14 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
*/
public StreamsProviderTask(StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
super(streamConfig);
+ streamConfig = super.streamConfig;
this.provider = provider;
if( perpetual )
this.type = Type.PERPETUAL;
else
this.type = Type.READ_CURRENT;
- this.timeout = DEFAULT_TIMEOUT_MS;
- this.sleepTime = DEFAULT_SLEEP_TIME_MS;
+ this.timeout = super.streamConfig.getProviderTimeoutMs().intValue();
+ this.sleepTime = streamConfig.getBatchFrequencyMs();
}
/**
@@ -96,8 +97,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
this.provider = provider;
this.type = Type.READ_NEW;
this.sequence = sequence;
- this.timeout = DEFAULT_TIMEOUT_MS;
- this.sleepTime = DEFAULT_SLEEP_TIME_MS;
+ this.timeout = streamConfig.getProviderTimeoutMs().intValue();
+ this.sleepTime = streamConfig.getBatchFrequencyMs();
}
/**
@@ -113,8 +114,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
this.dateRange = new DateTime[2];
this.dateRange[START] = start;
this.dateRange[END] = end;
- this.timeout = DEFAULT_TIMEOUT_MS;
- this.sleepTime = DEFAULT_SLEEP_TIME_MS;
+ this.timeout = streamConfig.getProviderTimeoutMs().intValue();
+ this.sleepTime = streamConfig.getBatchFrequencyMs();
}
public void setTimeout(int timeout) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
index e98dc0b..5c14c1f 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -32,16 +32,6 @@ import java.util.concurrent.BlockingQueue;
* in local mode.
*/
public interface StreamsTask extends Runnable{
- /**
- * Represents the default time to sleep a task before polling for the next set of work items
- */
- static final long DEFAULT_SLEEP_TIME_MS = 5000;
-
- /**
- * Represents the default amount of time to wait for new data to flow through the system before assuming it should
- * shut down
- */
- static final int DEFAULT_TIMEOUT_MS = 100000; //Will result in a default timeout of 1 minute if used
/**
* Informs the task to stop. Tasks may or may not try to empty its inbound queue before halting.
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/4081987c/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
index 7671a2d..38e948e 100644
--- a/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
+++ b/streams-runtimes/streams-runtime-local/src/test/java/org/apache/streams/local/tasks/BasicTasksTest.java
@@ -18,6 +18,7 @@
package org.apache.streams.local.tasks;
+import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.local.counters.DatumStatusCounter;
import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -74,12 +75,7 @@ public class BasicTasksTest {
service.submit(task);
int attempts = 0;
while(outQueue.size() != numMessages) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- ++attempts;
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
if(attempts == 10) {
fail("Provider task failed to output "+numMessages+" in a timely fashion.");
}
@@ -90,7 +86,7 @@ public class BasicTasksTest {
service.shutdownNow();
fail("Service did not terminate.");
}
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ assertTrue("Task should have completed running in allotted time.", service.isTerminated());
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
@@ -112,11 +108,7 @@ public class BasicTasksTest {
service.submit(task);
int attempts = 0;
while(inQueue.size() != 0 && outQueue.size() != numMessages) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
++attempts;
if(attempts == 10) {
fail("Processor task failed to output "+numMessages+" in a timely fashion.");
@@ -129,9 +121,9 @@ public class BasicTasksTest {
service.shutdownNow();
fail("Service did not terminate.");
}
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ assertTrue("Task should have completed running in allotted time.", service.isTerminated());
} catch (InterruptedException e) {
- fail("Test Interupted.");
+ fail("Test Interrupted.");
}
assertEquals(numMessages, processor.getMessageCount());
assertEquals(numMessages, counter.getNumReceived());
@@ -163,11 +155,7 @@ public class BasicTasksTest {
service.submit(task);
int attempts = 0;
while(inQueue.size() != 0 ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
++attempts;
if(attempts == 10) {
fail("Processor task failed to output "+numMessages+" in a timely fashion.");
@@ -180,9 +168,9 @@ public class BasicTasksTest {
service.shutdownNow();
fail("Service did not terminate.");
}
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ assertTrue("Task should have completed running in allotted time.", service.isTerminated());
} catch (InterruptedException e) {
- fail("Test Interupted.");
+ fail("Test Interrupted.");
}
assertEquals(numMessages, writer.getDatumsCounted());
assertEquals(numMessages, counter.getNumReceived());
@@ -205,11 +193,7 @@ public class BasicTasksTest {
service.submit(task);
int attempts = 0;
while(outQueue.size() != incoming * numMessages ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
++attempts;
if(attempts == 10) {
assertEquals("Processor task failed to output " + (numMessages * incoming) + " in a timely fashion.", (numMessages * incoming), outQueue.size());
@@ -222,9 +206,9 @@ public class BasicTasksTest {
service.shutdownNow();
fail("Service did not terminate.");
}
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ assertTrue("Task should have completed running in allotted time.", service.isTerminated());
} catch (InterruptedException e) {
- fail("Test Interupted.");
+ fail("Test Interrupted.");
}
}
@@ -244,11 +228,7 @@ public class BasicTasksTest {
service.submit(task);
int attempts = 0;
while(inQueue.size() != 0 ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
++attempts;
if(attempts == 10) {
assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
@@ -262,9 +242,9 @@ public class BasicTasksTest {
service.shutdownNow();
fail("Service did not terminate.");
}
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ assertTrue("Task should have completed running in allotted time.", service.isTerminated());
} catch (InterruptedException e) {
- fail("Test Interupted.");
+ fail("Test Interrupted.");
}
assertEquals(numMessages, processor.getMessageCount());
assertEquals(numMessages, outQueue1.size());
@@ -286,11 +266,7 @@ public class BasicTasksTest {
service.submit(task);
int attempts = 0;
while(inQueue.size() != 0 ) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException e) {
- //Ignore
- }
+ Uninterruptibles.sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
++attempts;
if(attempts == 10) {
assertEquals("Processor task failed to output "+(numMessages)+" in a timely fashion.", 0, inQueue.size());
@@ -304,9 +280,9 @@ public class BasicTasksTest {
service.shutdownNow();
fail("Service did not terminate.");
}
- assertTrue("Task should have completed running in aloted time.", service.isTerminated());
+ assertTrue("Task should have completed running in allotted time.", service.isTerminated());
} catch (InterruptedException e) {
- fail("Test Interupted.");
+ fail("Test Interrupted.");
}
assertEquals(numMessages, processor.getMessageCount());
assertEquals(numMessages, outQueue1.size());
[6/7] incubator-streams git commit: improvements and new test for
STREAMS-325
Posted by sb...@apache.org.
improvements and new test for STREAMS-325
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/94c3fd5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/94c3fd5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/94c3fd5e
Branch: refs/heads/master
Commit: 94c3fd5ea577f0710f61e7eb4c5c9af57ec33001
Parents: ebeaa16
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Fri Jun 12 11:29:32 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Fri Jun 12 11:53:14 2015 -0500
----------------------------------------------------------------------
.../tasks/BroadcastMonitorThread.java | 96 ++++++--------------
.../tasks/BroadcastMonitorThreadTest.java | 15 ++-
.../local/builders/LocalStreamBuilder.java | 2 +-
3 files changed, 44 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c3fd5e/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index b9db630..20c605e 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -53,15 +53,13 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(BroadcastMonitorThread.class);
private static MBeanServer server;
- private long DEFAULT_WAIT_TIME = 30000;
- private long waitTime;
- private ObjectMapper objectMapper;
- private StreamsConfiguration streamConfig;
private MonitoringConfiguration configuration;
private URI broadcastURI = null;
private MessagePersister messagePersister;
private volatile boolean keepRunning;
+ private static ObjectMapper objectMapper = StreamsJacksonMapper.getInstance();
+
/**
* DEPRECATED
* Please initialize logging with monitoring object via typesafe
@@ -69,24 +67,25 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
*/
@Deprecated
public BroadcastMonitorThread(Map<String, Object> streamConfig) {
- this(new ObjectMapper().convertValue(streamConfig, StreamsConfiguration.class));
+ this(objectMapper.convertValue(streamConfig, MonitoringConfiguration.class));
}
public BroadcastMonitorThread(StreamsConfiguration streamConfig) {
- this(streamConfig, new ObjectMapper().convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
+ this(objectMapper.convertValue(streamConfig.getAdditionalProperties().get("monitoring"), MonitoringConfiguration.class));
}
- public BroadcastMonitorThread(StreamsConfiguration streamConfig, MonitoringConfiguration configuration) {
+ public BroadcastMonitorThread(MonitoringConfiguration configuration) {
- this.streamConfig = streamConfig;
this.configuration = configuration;
+ if( this.configuration == null )
+ this.configuration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration(StreamsConfigurator.getConfig().atPath("monitoring"));
LOGGER.info("BroadcastMonitorThread created");
- prepare();
-
initializeObjectMapper();
+ prepare();
+
LOGGER.info("BroadcastMonitorThread initialized");
}
@@ -97,7 +96,6 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
* POJOs which are generated from JSON schemas
*/
private void initializeObjectMapper() {
- objectMapper = StreamsJacksonMapper.getInstance();
SimpleModule simpleModule = new SimpleModule();
simpleModule.addDeserializer(MemoryUsageBroadcast.class, new MemoryUsageDeserializer());
@@ -142,7 +140,7 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
}
messagePersister.persistMessages(messages);
- Thread.sleep(waitTime);
+ Thread.sleep(configuration.getMonitoringBroadcastIntervalMs());
} catch (InterruptedException e) {
LOGGER.debug("Broadcast Monitor Interrupted!");
Thread.currentThread().interrupt();
@@ -154,60 +152,33 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
}
}
- /**
- * Go through streams config and set the broadcastURI (if present)
- */
- private void setBroadcastURI() {
- if(configuration != null &&
- configuration.getBroadcastURI() != null) {
- try {
- broadcastURI = new URI(configuration.getBroadcastURI());
- } catch (URISyntaxException e) {
- e.printStackTrace();
- }
- }
- }
-
- /**
- * Go through streams config and set the thread's wait time (if present)
- */
- private void setWaitTime() {
- try {
- if (configuration != null &&
- configuration.getMonitoringBroadcastIntervalMs() != null ) {
- waitTime = configuration.getMonitoringBroadcastIntervalMs();
- } else {
- waitTime = DEFAULT_WAIT_TIME;
- }
-
- //Shutdown
- if(waitTime == -1) {
- this.keepRunning = false;
- }
- } catch (Exception e) {
- LOGGER.error("Exception while trying to set default broadcast thread wait time: {}", e);
- }
- }
-
public void prepare() {
keepRunning = true;
- LOGGER.info("BroadcastMonitorThread setup " + configuration);
+ LOGGER.info("BroadcastMonitorThread setup " + this.configuration);
server = ManagementFactory.getPlatformMBeanServer();
- setBroadcastURI();
- setWaitTime();
+ if (this.configuration != null) {
+
+ try {
+ broadcastURI = new URI(configuration.getBroadcastURI());
+ } catch (Exception e) {
+ LOGGER.error("invalid URI: ", e);
+ }
- if( broadcastURI != null ) {
- if (broadcastURI.getScheme().equals("http")) {
- messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
- } else if (broadcastURI.getScheme().equals("udp")) {
- messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
+ if (broadcastURI != null) {
+ if (broadcastURI.getScheme().equals("http")) {
+ messagePersister = new BroadcastMessagePersister(broadcastURI.toString());
+ } else if (broadcastURI.getScheme().equals("udp")) {
+ messagePersister = new LogstashUdpMessagePersister(broadcastURI.toString());
+ } else {
+ LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
+ throw new RuntimeException();
+ }
} else {
- LOGGER.error("You need to specify a broadcast URI with either a HTTP or UDP protocol defined.");
- throw new RuntimeException();
+ messagePersister = new SLF4JMessagePersister();
}
} else {
messagePersister = new SLF4JMessagePersister();
@@ -221,18 +192,11 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
}
public String getBroadcastURI() {
- return broadcastURI.toString();
+ return configuration.getBroadcastURI();
}
public long getWaitTime() {
- return waitTime;
- }
-
- public long getDefaultWaitTime() {
- return DEFAULT_WAIT_TIME;
+ return configuration.getMonitoringBroadcastIntervalMs();
}
- public void setDefaultWaitTime(long defaultWaitTime) {
- this.DEFAULT_WAIT_TIME = defaultWaitTime;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c3fd5e/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
index 427757b..a959bd2 100644
--- a/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
+++ b/streams-monitoring/src/test/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThreadTest.java
@@ -52,9 +52,20 @@ public class BroadcastMonitorThreadTest {
testThread(thread);
}
+ @Test
+ public void testThreadStreamsConfig() {
+
+ StreamsConfiguration streams = new StreamsConfiguration();
+ MonitoringConfiguration monitoring = new MonitoringConfiguration();
+ monitoring.setBroadcastURI("http://fakeurl.com/fake");
+ monitoring.setMonitoringBroadcastIntervalMs(30000L);
+ streams.setAdditionalProperty("monitoring", monitoring);
+ BroadcastMonitorThread thread = new BroadcastMonitorThread(streams);
+ testThread(thread);
+ }
+
public void testThread(BroadcastMonitorThread thread) {
- thread.setDefaultWaitTime(3000L);
- long testRunLength = thread.getDefaultWaitTime() * 1;
+ long testRunLength = thread.getWaitTime() * 1;
executor = Executors.newFixedThreadPool(1);
executor.submit(thread);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/94c3fd5e/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 4175e10..4cf078e 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -151,7 +151,7 @@ public class LocalStreamBuilder implements StreamBuilder {
}
};
this.useDeprecatedMonitors = false;
- this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
+ this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig.getMonitoring());
}
public void setUseDeprecatedMonitors(boolean useDeprecatedMonitors) {
[5/7] incubator-streams git commit: fix one reference to removed
config field adapts dropwizard runtime configuration
Posted by sb...@apache.org.
fix one reference to removed config field
adapts dropwizard runtime configuration
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/ebeaa162
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/ebeaa162
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/ebeaa162
Branch: refs/heads/master
Commit: ebeaa1622a1e1235d17a59d20ab0dabe1456fe7e
Parents: 4081987
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Mon May 25 22:08:49 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Mon May 25 22:08:49 2015 -0500
----------------------------------------------------------------------
.../apache/streams/dropwizard/StreamDropwizardBuilder.java | 8 ++++++++
.../org/apache/streams/dropwizard/StreamsApplication.java | 6 +-----
.../org/apache/streams/local/tasks/StreamsMergeTask.java | 8 ++------
3 files changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ebeaa162/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
index d409a4a..0fcc4eb 100644
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
+++ b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamDropwizardBuilder.java
@@ -18,8 +18,12 @@
package org.apache.streams.dropwizard;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.streams.config.StreamsConfiguration;
+import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.StreamBuilder;
import org.apache.streams.core.StreamsProvider;
+import org.apache.streams.local.LocalRuntimeConfiguration;
import org.apache.streams.local.builders.LocalStreamBuilder;
import java.util.Map;
@@ -35,6 +39,10 @@ public class StreamDropwizardBuilder extends LocalStreamBuilder implements Strea
super();
}
+ public StreamDropwizardBuilder(StreamsConfiguration streamConfig) {
+ super(new ObjectMapper().convertValue(streamConfig, LocalRuntimeConfiguration.class));
+ }
+
public StreamDropwizardBuilder(Map<String, Object> streamConfig) {
super(streamConfig);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ebeaa162/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
index 991d900..463320c 100644
--- a/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
+++ b/streams-runtimes/streams-runtime-dropwizard/src/main/java/org/apache/streams/dropwizard/StreamsApplication.java
@@ -145,11 +145,7 @@ public class StreamsApplication extends Application<StreamsDropwizardConfigurati
public StreamBuilder setup(StreamsConfiguration streamsConfiguration, Set<StreamsProvider> resourceProviders) {
- Map<String, Object> streamConfig = Maps.newHashMap();
- streamConfig.put(LocalStreamBuilder.TIMEOUT_KEY, streamsConfiguration.getTimeoutMs().intValue());
- if(! Strings.isNullOrEmpty(streamsConfiguration.getBroadcastURI()) ) streamConfig.put("broadcastURI", streamsConfiguration.getBroadcastURI());
- if(streamsConfiguration.getBroadcastIntervalMs() != null ) streamConfig.put("monitoring_broadcast_interval_ms", streamsConfiguration.getBroadcastIntervalMs().intValue());
- StreamBuilder builder = new StreamDropwizardBuilder(1000, streamConfig);
+ StreamBuilder builder = new StreamDropwizardBuilder(streamsConfiguration);
List<String> providers = new ArrayList<>();
for( StreamsProvider provider: resourceProviders) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/ebeaa162/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
index 49cb074..69cd5a5 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -38,14 +38,10 @@ public class StreamsMergeTask extends BaseStreamsTask {
private long sleepTime;
public StreamsMergeTask() {
- this(DEFAULT_SLEEP_TIME_MS, null);
+ this(null);
}
- public StreamsMergeTask(long sleepTime) {
- this(sleepTime, null);
- }
-
- public StreamsMergeTask(long sleepTime, StreamsConfiguration streamConfig) {
+ public StreamsMergeTask(StreamsConfiguration streamConfig) {
super(streamConfig);
this.sleepTime = sleepTime;
this.keepRunning = new AtomicBoolean(true);
[7/7] incubator-streams git commit: Merge commit
'94c3fd5ea577f0710f61e7eb4c5c9af57ec33001'
Posted by sb...@apache.org.
Merge commit '94c3fd5ea577f0710f61e7eb4c5c9af57ec33001'
* commit '94c3fd5ea577f0710f61e7eb4c5c9af57ec33001':
improvements and new test for STREAMS-325
fix one reference to removed config field adapts dropwizard runtime configuration
completed, all tests passing resolves #226 resolves #294 resolves #325
initial refactoring
resolves STREAMS-325 #325
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/39c07f1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/39c07f1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/39c07f1d
Branch: refs/heads/master
Commit: 39c07f1d64757626acf496028a0367fa5d347ff0
Parents: e7bf641 94c3fd5
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Fri Jun 19 16:44:46 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Fri Jun 19 16:44:46 2015 -0500
----------------------------------------------------------------------
.../main/jsonschema/StreamsConfiguration.json | 38 ++++--
streams-monitoring/pom.xml | 11 +-
.../tasks/BroadcastMonitorThread.java | 124 +++++++++----------
.../monitoring/MonitoringConfiguration.json | 23 ++++
.../tasks/BroadcastMonitorThreadTest.java | 38 +++++-
.../dropwizard/StreamDropwizardBuilder.java | 8 ++
.../streams/dropwizard/StreamsApplication.java | 6 +-
.../local/builders/LocalStreamBuilder.java | 75 +++++++----
.../streams/local/builders/StreamComponent.java | 13 +-
.../streams/local/tasks/BaseStreamsTask.java | 41 ++----
.../streams/local/tasks/StreamsMergeTask.java | 11 +-
.../local/tasks/StreamsPersistWriterTask.java | 22 ++--
.../local/tasks/StreamsProcessorTask.java | 22 ++--
.../local/tasks/StreamsProviderTask.java | 24 ++--
.../apache/streams/local/tasks/StreamsTask.java | 13 +-
.../jsonschema/LocalRuntimeConfiguration.json | 8 +-
.../streams/local/tasks/BasicTasksTest.java | 60 +++------
17 files changed, 286 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
[3/7] incubator-streams git commit: Merge branch 'STREAMS-325' into
STREAMS-226
Posted by sb...@apache.org.
Merge branch 'STREAMS-325' into STREAMS-226
* STREAMS-325:
resolves STREAMS-325 #325
Conflicts:
streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/40b9b1c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/40b9b1c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/40b9b1c5
Branch: refs/heads/master
Commit: 40b9b1c58796a68f0ccf14ff20915a0fac66f812
Parents: 24da4ca 8bd2255
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Sun May 24 19:09:18 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Sun May 24 19:09:18 2015 -0500
----------------------------------------------------------------------
streams-monitoring/pom.xml | 11 +--
.../tasks/BroadcastMonitorThread.java | 83 ++++++++++++--------
.../monitoring/MonitoringConfiguration.json | 23 ++++++
.../tasks/BroadcastMonitorThreadTest.java | 24 ++++--
.../local/builders/LocalStreamBuilder.java | 19 ++++-
5 files changed, 117 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/40b9b1c5/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --cc streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index aaf649c,801d2da..5d0377a
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@@ -209,4 -228,4 +228,4 @@@ public class BroadcastMonitorThread ext
public void setDefaultWaitTime(long defaultWaitTime) {
this.DEFAULT_WAIT_TIME = defaultWaitTime;
}
--}
++}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/40b9b1c5/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --cc streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 5403130,f6d8f23..59d60df
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@@ -19,10 -19,12 +19,13 @@@
package org.apache.streams.local.builders;
import com.google.common.util.concurrent.Uninterruptibles;
+ import org.apache.streams.config.ComponentConfigurator;
+ import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.*;
+import org.apache.streams.local.LocalRuntimeConfiguration;
import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
+ import org.apache.streams.local.monitoring.MonitoringConfiguration;
import org.apache.streams.local.queues.ThroughputQueue;
import org.apache.streams.local.tasks.*;
import org.apache.streams.monitoring.tasks.BroadcastMonitorThread;
@@@ -127,10 -120,24 +130,24 @@@ public class LocalStreamBuilder impleme
setStreamIdentifier();
if(this.streamConfig != null) {
- this.streamConfig.put(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
+ this.streamConfig.setAdditionalProperty(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
}
this.useDeprecatedMonitors = false;
- this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
+
+ /* for backward-compatibility with streamConfig */
+ MonitoringConfiguration monitoringConfiguration;
+ if( StreamsConfigurator.getConfig().hasPath("monitoring") ) {
+ monitoringConfiguration = new ComponentConfigurator<>(MonitoringConfiguration.class).detectConfiguration("monitoring");
+ } else {
+ monitoringConfiguration = new MonitoringConfiguration();
+ }
+ if( this.streamConfig != null &&
+ this.streamConfig.containsKey(BROADCAST_KEY)) {
+ monitoringConfiguration.setBroadcastURI(this.streamConfig.get(BROADCAST_KEY).toString());
+ if( this.streamConfig.containsKey(BROADCAST_INTERVAL_KEY))
+ monitoringConfiguration.setMonitoringBroadcastIntervalMs(Long.parseLong(this.streamConfig.get(BROADCAST_INTERVAL_KEY).toString()));
+ }
+ this.broadcastMonitor = new BroadcastMonitorThread(monitoringConfiguration);
this.futures = new HashMap<>();
}
[2/7] incubator-streams git commit: initial refactoring
Posted by sb...@apache.org.
initial refactoring
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/24da4cab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/24da4cab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/24da4cab
Branch: refs/heads/master
Commit: 24da4cab0012fb2ae3ef04cd22971c978dc5106b
Parents: 6a05779
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Sun May 24 19:04:36 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Sun May 24 19:04:36 2015 -0500
----------------------------------------------------------------------
.../main/jsonschema/StreamsConfiguration.json | 38 +++++++++++++++-----
.../tasks/BroadcastMonitorThread.java | 2 +-
.../local/builders/LocalStreamBuilder.java | 27 +++++++-------
.../streams/local/builders/StreamComponent.java | 13 +++----
.../streams/local/tasks/BaseStreamsTask.java | 23 +++---------
.../streams/local/tasks/StreamsMergeTask.java | 5 +--
.../local/tasks/StreamsPersistWriterTask.java | 9 ++---
.../local/tasks/StreamsProcessorTask.java | 11 +++---
.../local/tasks/StreamsProviderTask.java | 11 +++---
.../apache/streams/local/tasks/StreamsTask.java | 3 +-
.../jsonschema/LocalRuntimeConfiguration.json | 8 +++--
11 files changed, 85 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-config/src/main/jsonschema/StreamsConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-config/src/main/jsonschema/StreamsConfiguration.json b/streams-config/src/main/jsonschema/StreamsConfiguration.json
index 2f68164..bfff501 100644
--- a/streams-config/src/main/jsonschema/StreamsConfiguration.json
+++ b/streams-config/src/main/jsonschema/StreamsConfiguration.json
@@ -8,20 +8,40 @@
"javaType" : "org.apache.streams.config.StreamsConfiguration",
"javaInterfaces": ["java.io.Serializable"],
"properties": {
- "pipeline": {
+ "batchSize": {
+ "type" : "integer",
+ "description" : "Default max batch size for batch-based persist readers and writers",
+ "default": 1000
+ },
+ "identifier": {
"type" : "string",
+ "description" : "Name of streams pipeline",
"default": "No-name Stream"
},
- "broadcastURI": {
- "type": "string"
+ "parallelism": {
+ "type" : "integer",
+ "description" : "Default parallelism for processors",
+ "default": 1
+ },
+ "queueSize": {
+ "type" : "integer",
+ "description" : "Size of inter-component buffers",
+ "default": 1000
+ },
+ "batchFrequencyMs": {
+ "type": "integer",
+ "description": "Frequency of messaging between components",
+ "default": 5000
},
- "broadcastIntervalMs": {
- "type": "integer"
+ "providerTimeoutMs": {
+ "type": "integer",
+ "description": "Kill the stream if all providers supply nothing for this long.",
+ "default": 1440000
},
- "timeoutMs": {
- "type": "integer",
- "default": "1440000",
- "description": "Kill the stream if all providers supply nothing for this long."
+ "startedAt": {
+ "type": "integer",
+ "description": "Timestamp of beginning of stream execution",
+ "default": -1
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
----------------------------------------------------------------------
diff --git a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
index 19b1067..aaf649c 100644
--- a/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
+++ b/streams-monitoring/src/main/java/org/apache/streams/monitoring/tasks/BroadcastMonitorThread.java
@@ -52,7 +52,7 @@ public class BroadcastMonitorThread extends NotificationBroadcasterSupport imple
private long DEFAULT_WAIT_TIME = 30000;
private long waitTime;
private ObjectMapper objectMapper;
- private Map<String, Object> streamConfig;
+ private StreamsCon streamConfig;
private URI broadcastURI = null;
private MessagePersister messagePersister;
private volatile boolean keepRunning;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 601c6b7..5403130 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -20,6 +20,7 @@ package org.apache.streams.local.builders;
import com.google.common.util.concurrent.Uninterruptibles;
import org.apache.streams.core.*;
+import org.apache.streams.local.LocalRuntimeConfiguration;
import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.streams.local.executors.ShutdownStreamOnUnhandleThrowableThreadPoolExecutor;
import org.apache.streams.local.queues.ThroughputQueue;
@@ -54,7 +55,7 @@ public class LocalStreamBuilder implements StreamBuilder {
private Map<String, StreamComponent> providers;
private Map<String, StreamComponent> components;
- private Map<String, Object> streamConfig;
+ private LocalRuntimeConfiguration streamConfig;
private Map<StreamsTask, Future> futures;
private ExecutorService executor;
private ExecutorService monitor;
@@ -79,7 +80,9 @@ public class LocalStreamBuilder implements StreamBuilder {
/**
* Creates a local stream builder with a config object and default maximum internal queue size of 500
* @param streamConfig
+ * @deprecated use LocalRuntimeConfiguration constructor instread
*/
+ @Deprecated
public LocalStreamBuilder(Map<String, Object> streamConfig) {
this(DEFAULT_QUEUE_SIZE, streamConfig);
}
@@ -100,12 +103,19 @@ public class LocalStreamBuilder implements StreamBuilder {
* @param streamConfig
*/
public LocalStreamBuilder(int maxQueueCapacity, Map<String, Object> streamConfig) {
+ LocalRuntimeConfiguration config = new LocalRuntimeConfiguration();
+ for( Map.Entry<String, Object> item : streamConfig.entrySet() ) {
+ config.setAdditionalProperty(item.getKey(), item.getValue());
+ }
+ config.setBatchSize(new Long(maxQueueCapacity));
+ }
+
+ public LocalStreamBuilder(LocalRuntimeConfiguration streamConfig) {
this.providers = new HashMap<String, StreamComponent>();
this.components = new HashMap<String, StreamComponent>();
this.streamConfig = streamConfig;
this.totalTasks = 0;
this.monitorTasks = 0;
- this.maxQueueCapacity = maxQueueCapacity;
final LocalStreamBuilder self = this;
this.shutdownHook = new Thread() {
@Override
@@ -117,7 +127,7 @@ public class LocalStreamBuilder implements StreamBuilder {
setStreamIdentifier();
if(this.streamConfig != null) {
- this.streamConfig.put(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
+ this.streamConfig.setAdditionalProperty(DEFAULT_STARTED_AT_KEY, startedAt.getMillis());
}
this.useDeprecatedMonitors = false;
this.broadcastMonitor = new BroadcastMonitorThread(this.streamConfig);
@@ -435,18 +445,11 @@ public class LocalStreamBuilder implements StreamBuilder {
protected int getTimeout() {
//Set the timeout of it is configured, otherwise signal downstream components to use their default
- return streamConfig != null && streamConfig.containsKey(TIMEOUT_KEY) ? (Integer)streamConfig.get(TIMEOUT_KEY) : -1;
+ return streamConfig.getProviderTimeoutMs().intValue();
}
private void setStreamIdentifier() {
- if(streamConfig != null &&
- streamConfig.containsKey(STREAM_IDENTIFIER_KEY) &&
- streamConfig.get(STREAM_IDENTIFIER_KEY) != null &&
- streamConfig.get(STREAM_IDENTIFIER_KEY).toString().length() > 0) {
- this.streamIdentifier = streamConfig.get(STREAM_IDENTIFIER_KEY).toString();
- } else {
- this.streamIdentifier = DEFAULT_STREAM_IDENTIFIER;
- }
+ this.streamIdentifier = streamConfig.getIdentifier();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
index 31c5981..4f2a507 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/StreamComponent.java
@@ -19,6 +19,7 @@
package org.apache.streams.local.builders;
import com.google.common.collect.Lists;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.*;
import org.apache.streams.local.tasks.StreamsPersistWriterTask;
import org.apache.streams.local.tasks.StreamsProcessorTask;
@@ -54,14 +55,14 @@ public class StreamComponent {
private List<StreamsTask> tasks;
- private Map<String, Object> streamConfig;
+ private StreamsConfiguration streamConfig;
/**
*
* @param id
* @param provider
*/
- public StreamComponent(String id, StreamsProvider provider, boolean perpetual, Map<String, Object> streamConfig) {
+ public StreamComponent(String id, StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
this.id = id;
this.provider = provider;
this.perpetual = perpetual;
@@ -76,7 +77,7 @@ public class StreamComponent {
* @param start
* @param end
*/
- public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end, Map<String, Object> streamConfig) {
+ public StreamComponent(String id, StreamsProvider provider, DateTime start, DateTime end, StreamsConfiguration streamConfig) {
this.id = id;
this.provider = provider;
this.dateRange = new DateTime[2];
@@ -93,7 +94,7 @@ public class StreamComponent {
* @param provider
* @param sequence
*/
- public StreamComponent(String id, StreamsProvider provider, BigInteger sequence, Map<String, Object> streamConfig) {
+ public StreamComponent(String id, StreamsProvider provider, BigInteger sequence, StreamsConfiguration streamConfig) {
this.id = id;
this.provider = provider;
this.sequence = sequence;
@@ -107,7 +108,7 @@ public class StreamComponent {
* @param inQueue
* @param numTasks
*/
- public StreamComponent(String id, StreamsProcessor processor, BlockingQueue<StreamsDatum> inQueue, int numTasks, Map<String, Object> streamConfig) {
+ public StreamComponent(String id, StreamsProcessor processor, BlockingQueue<StreamsDatum> inQueue, int numTasks, StreamsConfiguration streamConfig) {
this.id = id;
this.processor = processor;
this.inQueue = inQueue;
@@ -123,7 +124,7 @@ public class StreamComponent {
* @param inQueue
* @param numTasks
*/
- public StreamComponent(String id, StreamsPersistWriter writer, BlockingQueue<StreamsDatum> inQueue, int numTasks, Map<String, Object> streamConfig) {
+ public StreamComponent(String id, StreamsPersistWriter writer, BlockingQueue<StreamsDatum> inQueue, int numTasks, StreamsConfiguration streamConfig) {
this.id = id;
this.writer = writer;
this.inQueue = inQueue;
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
index c0c7ea6..3daa110 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/BaseStreamsTask.java
@@ -21,6 +21,7 @@ package org.apache.streams.local.tasks;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.jackson.StreamsJacksonMapper;
import org.apache.streams.local.builders.LocalStreamBuilder;
@@ -46,12 +47,12 @@ public abstract class BaseStreamsTask implements StreamsTask {
private List<BlockingQueue<StreamsDatum>> outQueues = new LinkedList<BlockingQueue<StreamsDatum>>();
private int inIndex = 0;
private ObjectMapper mapper;
- protected Map<String, Object> streamConfig;
+ protected StreamsConfiguration streamConfig;
private long startedAt;
private String streamIdentifier;
- public BaseStreamsTask(Map<String, Object> config) {
+ public BaseStreamsTask(StreamsConfiguration config) {
this.mapper = StreamsJacksonMapper.getInstance();
this.mapper.registerSubtypes(Activity.class);
this.streamConfig = config;
@@ -205,14 +206,7 @@ public abstract class BaseStreamsTask implements StreamsTask {
}
public void setStartedAt() {
- if(streamConfig != null &&
- streamConfig.containsKey(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY) &&
- streamConfig.get(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY) != null &&
- streamConfig.get(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY) instanceof Long) {
- this.startedAt = Long.parseLong(streamConfig.get(LocalStreamBuilder.DEFAULT_STARTED_AT_KEY).toString());
- } else {
- this.startedAt = -1;
- }
+ this.startedAt = streamConfig.getStartedAt();
}
public String getStreamIdentifier() {
@@ -220,14 +214,7 @@ public abstract class BaseStreamsTask implements StreamsTask {
}
public void setStreamIdentifier() {
- if(streamConfig != null &&
- streamConfig.containsKey(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) &&
- streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY) != null &&
- streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY).toString().length() > 0) {
- this.streamIdentifier = streamConfig.get(LocalStreamBuilder.STREAM_IDENTIFIER_KEY).toString();
- } else {
- this.streamIdentifier = LocalStreamBuilder.DEFAULT_STREAM_IDENTIFIER;
- }
+ this.streamIdentifier = streamConfig.getIdentifier();
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
index ed38fe8..49cb074 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsMergeTask.java
@@ -18,6 +18,7 @@
package org.apache.streams.local.tasks;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.local.counters.StreamsTaskCounter;
import org.apache.commons.lang.NotImplementedException;
@@ -44,7 +45,7 @@ public class StreamsMergeTask extends BaseStreamsTask {
this(sleepTime, null);
}
- public StreamsMergeTask(long sleepTime, Map<String, Object> streamConfig) {
+ public StreamsMergeTask(long sleepTime, StreamsConfiguration streamConfig) {
super(streamConfig);
this.sleepTime = sleepTime;
this.keepRunning = new AtomicBoolean(true);
@@ -56,7 +57,7 @@ public class StreamsMergeTask extends BaseStreamsTask {
}
@Override
- public void setStreamConfig(Map<String, Object> config) {
+ public void setStreamConfig(StreamsConfiguration config) {
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index 860a8f0..d6bf2ec 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -19,6 +19,7 @@
package org.apache.streams.local.tasks;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -40,7 +41,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
private StreamsPersistWriter writer;
private long sleepTime = DEFAULT_SLEEP_TIME_MS * 10;
private AtomicBoolean keepRunning;
- private Map<String, Object> streamConfig;
+ private StreamsConfiguration streamConfig;
private BlockingQueue<StreamsDatum> inQueue;
private AtomicBoolean isRunning;
private AtomicBoolean blocked;
@@ -62,7 +63,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
this(writer, DEFAULT_SLEEP_TIME_MS, null);
}
- public StreamsPersistWriterTask(StreamsPersistWriter writer, Map<String, Object> streamConfig) {
+ public StreamsPersistWriterTask(StreamsPersistWriter writer, StreamsConfiguration streamConfig) {
this(writer, DEFAULT_SLEEP_TIME_MS, streamConfig);
}
@@ -71,7 +72,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
* @param writer writer to execute in task
* @param sleepTime time to sleep when inbound queue is empty.
*/
- public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime, Map<String, Object> streamConfig) {
+ public StreamsPersistWriterTask(StreamsPersistWriter writer, long sleepTime, StreamsConfiguration streamConfig) {
super(streamConfig);
this.writer = writer;
this.sleepTime = sleepTime;
@@ -86,7 +87,7 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
}
@Override
- public void setStreamConfig(Map<String, Object> config) {
+ public void setStreamConfig(StreamsConfiguration config) {
this.streamConfig = config;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index d03dfa3..5e04b2f 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -19,6 +19,7 @@
package org.apache.streams.local.tasks;
import com.google.common.collect.Maps;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -41,7 +42,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
private StreamsProcessor processor;
private long sleepTime;
private AtomicBoolean keepRunning;
- private Map<String, Object> streamConfig;
+ private StreamsConfiguration streamConfig;
private BlockingQueue<StreamsDatum> inQueue;
private AtomicBoolean isRunning;
private AtomicBoolean blocked;
@@ -67,14 +68,14 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
* @param processor
* @param streamConfig
*/
- public StreamsProcessorTask(StreamsProcessor processor, Map<String, Object> streamConfig) { this(processor, DEFAULT_SLEEP_TIME_MS, streamConfig); }
+ public StreamsProcessorTask(StreamsProcessor processor, StreamsConfiguration streamConfig) { this(processor, DEFAULT_SLEEP_TIME_MS, streamConfig); }
/**
*
* @param processor processor to run in task
* @param sleepTime time to sleep when incoming queue is empty
*/
- public StreamsProcessorTask(StreamsProcessor processor, long sleepTime, Map<String, Object> streamConfig) {
+ public StreamsProcessorTask(StreamsProcessor processor, long sleepTime, StreamsConfiguration streamConfig) {
super(streamConfig);
this.processor = processor;
this.sleepTime = sleepTime;
@@ -94,7 +95,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
}
@Override
- public void setStreamConfig(Map<String, Object> config) {
+ public void setStreamConfig(StreamsConfiguration config) {
this.streamConfig = config;
}
@@ -119,7 +120,7 @@ public class StreamsProcessorTask extends BaseStreamsTask implements DatumStatus
StreamsDatum datum = null;
try {
this.blocked.set(true);
- datum = this.inQueue.poll(5, TimeUnit.SECONDS);
+ datum = this.inQueue.poll(streamConfig.getBatchFrequencyMs(), TimeUnit.MILLISECONDS);
} catch (InterruptedException ie) {
LOGGER.debug("Received InteruptedException, shutting down and re-applying interrupt status.");
this.keepRunning.set(false);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index c446a72..cb36a24 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -19,6 +19,7 @@
package org.apache.streams.local.tasks;
import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.*;
import org.apache.streams.core.util.DatumUtils;
import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -62,7 +63,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
private Type type;
private BigInteger sequence;
private DateTime[] dateRange;
- private Map<String, Object> config;
+ private StreamsConfiguration config;
private int timeout;
private long sleepTime;
@@ -74,7 +75,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
* Constructor for a StreamsProvider to execute {@link org.apache.streams.core.StreamsProvider:readCurrent()}
* @param provider
*/
- public StreamsProviderTask(StreamsProvider provider, boolean perpetual, Map<String, Object> streamConfig) {
+ public StreamsProviderTask(StreamsProvider provider, boolean perpetual, StreamsConfiguration streamConfig) {
super(streamConfig);
this.provider = provider;
if( perpetual )
@@ -90,7 +91,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
* @param provider
* @param sequence
*/
- public StreamsProviderTask(StreamsProvider provider, BigInteger sequence, Map<String, Object> streamConfig) {
+ public StreamsProviderTask(StreamsProvider provider, BigInteger sequence, StreamsConfiguration streamConfig) {
super(streamConfig);
this.provider = provider;
this.type = Type.READ_NEW;
@@ -105,7 +106,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
* @param start
* @param end
*/
- public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end, Map<String, Object> streamConfig) {
+ public StreamsProviderTask(StreamsProvider provider, DateTime start, DateTime end, StreamsConfiguration streamConfig) {
super(streamConfig);
this.provider = provider;
this.type = Type.READ_RANGE;
@@ -141,7 +142,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
}
@Override
- public void setStreamConfig(Map<String, Object> config) {
+ public void setStreamConfig(StreamsConfiguration config) {
this.config = config;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
index 8423095..e98dc0b 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsTask.java
@@ -18,6 +18,7 @@
package org.apache.streams.local.tasks;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.core.StreamsDatum;
import org.apache.streams.local.counters.StreamsTaskCounter;
@@ -68,7 +69,7 @@ public interface StreamsTask extends Runnable{
* Set the configuration object that will shared and passed to all instances of StreamsTask.
* @param config optional configuration information
*/
- public void setStreamConfig(Map<String, Object> config);
+ public void setStreamConfig(StreamsConfiguration config);
/**
* Returns true when the task has not completed. Returns false otherwise
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/24da4cab/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
index c7f79a0..36177d9 100644
--- a/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
+++ b/streams-runtimes/streams-runtime-local/src/main/jsonschema/LocalRuntimeConfiguration.json
@@ -6,10 +6,14 @@
"id": "#",
"type": "object",
"javaType" : "org.apache.streams.local.LocalRuntimeConfiguration",
+ "extends": {
+ "$ref": "../../../../../streams-config/src/main/jsonschema/StreamsConfiguration.json"
+ },
"javaInterfaces": ["java.io.Serializable"],
"properties": {
- "providerTimeoutMs": {
- "type" : "integer"
+ "monitoring": {
+ "type" : "object",
+ "javaType" : "org.apache.streams.local.monitoring.MonitoringConfiguration"
}
}
}
\ No newline at end of file