You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/05/13 17:22:51 UTC
[01/14] git commit: userstream now writing to ES in real-time
Repository: incubator-streams
Updated Branches:
refs/heads/master c38eef6cf -> e68300936
userstream now writing to ES in real-time
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/522096e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/522096e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/522096e8
Branch: refs/heads/master
Commit: 522096e8a914ea738a97a536ef19a1416453e5f0
Parents: f877c5f
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Mon Apr 14 10:25:27 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Mon Apr 14 10:25:27 2014 -0500
----------------------------------------------------------------------
.../streams-persist-elasticsearch/pom.xml | 1 -
.../ElasticsearchConfigurator.java | 7 ++
.../ElasticsearchPersistWriter.java | 25 ++++---
.../ElasticsearchWriterConfiguration.json | 10 +++
.../twitter/processor/TwitterTypeConverter.java | 2 +-
.../twitter/provider/TwitterStreamProvider.java | 69 +++++++++-----------
.../provider/TwitterTimelineProvider.java | 32 ++++-----
.../local/tasks/StreamsProviderTask.java | 2 +-
8 files changed, 78 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/pom.xml b/streams-contrib/streams-persist-elasticsearch/pom.xml
index 07433ba..95f3357 100644
--- a/streams-contrib/streams-persist-elasticsearch/pom.xml
+++ b/streams-contrib/streams-persist-elasticsearch/pom.xml
@@ -62,7 +62,6 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
- <version>1.8</version>
<executions>
<execution>
<id>add-source</id>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 20b5c08..e6bf3ca 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@ -52,9 +52,16 @@ public class ElasticsearchConfigurator {
String index = elasticsearch.getString("index");
String type = elasticsearch.getString("type");
+ if( elasticsearch.hasPath("bulk"))
+ elasticsearchWriterConfiguration.setBulk(elasticsearch.getBoolean("bulk"));
+
+ if( elasticsearch.hasPath("batchSize"))
+ elasticsearchWriterConfiguration.setBatchSize(elasticsearch.getLong("batchSize"));
+
elasticsearchWriterConfiguration.setIndex(index);
elasticsearchWriterConfiguration.setType(type);
+
return elasticsearchWriterConfiguration;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ad89013..405d1b8 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -51,34 +51,27 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private BulkRequestBuilder bulkRequest;
private OutputStreamWriter currentWriter = null;
- private int batchSize = 50;
- private int totalRecordsWritten = 0;
- private boolean veryLargeBulk = false; // by default this setting is set to false
+ private long batchSize;
+ private boolean veryLargeBulk; // by default this setting is set to false
private final static Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
private static final long WAITING_DOCS_LIMIT = 10000;
public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
+ private volatile int currentItems = 0;
private volatile int totalSent = 0;
private volatile int totalSeconds = 0;
private volatile int totalAttempted = 0;
private volatile int totalOk = 0;
private volatile int totalFailed = 0;
private volatile int totalBatchCount = 0;
+ private volatile int totalRecordsWritten = 0;
private volatile long totalSizeInBytes = 0;
private volatile long batchSizeInBytes = 0;
private volatile int batchItemsSent = 0;
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
- }
-
- public void setVeryLargeBulk(boolean veryLargeBulk) {
- this.veryLargeBulk = veryLargeBulk;
- }
-
private final List<String> affectedIndexes = new ArrayList<String>();
public int getTotalOutstanding() { return this.totalSent - (this.totalFailed + this.totalOk); }
@@ -99,7 +92,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
protected volatile Queue<StreamsDatum> persistQueue;
- private ObjectMapper mapper = new StreamsJacksonMapper();
+ private ObjectMapper mapper;
private ElasticsearchWriterConfiguration config;
@@ -245,6 +238,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
// reset the current batch statistics
this.batchSizeInBytes = 0;
this.batchItemsSent = 0;
+ this.currentItems = 0;
try
{
@@ -323,6 +317,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+
}
@Override
@@ -409,11 +404,13 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private void trackItemAndBytesWritten(long sizeInBytes)
{
+ currentItems++;
batchItemsSent++;
batchSizeInBytes += sizeInBytes;
// If our queue is larger than our flush threashold, then we should flush the queue.
- if(batchSizeInBytes > flushThresholdSizeInBytes)
+ if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+ (currentItems >= batchSize) )
flushInternal();
}
@@ -530,6 +527,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
@Override
public void prepare(Object configurationObject) {
mapper = StreamsJacksonMapper.getInstance();
+ veryLargeBulk = this.config.getBulk();
+ batchSize = this.config.getBatchSize();
start();
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 21aad5c..5e169b7 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@ -13,6 +13,16 @@
"type": {
"type": "string",
"description": "Type to write as"
+ },
+ "bulk": {
+ "type": "boolean",
+ "description": "Index in large or small batches",
+ "default": "false"
+ },
+ "batchSize": {
+ "type": "integer",
+ "description": "Item Count before flush",
+ "default": 100
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
index 68820c9..5e88bdd 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
@@ -27,7 +27,7 @@ import java.util.Queue;
*/
public class TwitterTypeConverter implements StreamsProcessor {
- private final static String STREAMS_ID = "TwitterTypeConverter";
+ public final static String STREAMS_ID = "TwitterTypeConverter";
private final static Logger LOGGER = LoggerFactory.getLogger(TwitterTypeConverter.class);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
index 3df7d02..ba88803 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
@@ -1,10 +1,7 @@
package org.apache.streams.twitter.provider;
-import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
import com.google.common.collect.Queues;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@@ -19,10 +16,9 @@ import com.twitter.hbc.httpclient.auth.Authentication;
import com.twitter.hbc.httpclient.auth.BasicAuth;
import com.twitter.hbc.httpclient.auth.OAuth1;
import com.typesafe.config.Config;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.core.*;
import org.apache.streams.twitter.TwitterStreamConfiguration;
import org.apache.streams.twitter.processor.TwitterEventProcessor;
import org.joda.time.DateTime;
@@ -31,7 +27,6 @@ import org.slf4j.LoggerFactory;
import java.io.Serializable;
import java.math.BigInteger;
-import java.util.Collection;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.*;
@@ -39,7 +34,7 @@ import java.util.concurrent.*;
/**
* Created by sblackmon on 12/10/13.
*/
-public class TwitterStreamProvider implements StreamsProvider, Serializable {
+public class TwitterStreamProvider implements StreamsProvider, Serializable, DatumStatusCountable {
public final static String STREAMS_ID = "TwitterStreamProvider";
@@ -47,8 +42,6 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
private TwitterStreamConfiguration config;
- private Class klass;
-
public TwitterStreamConfiguration getConfig() {
return config;
}
@@ -57,9 +50,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
this.config = config;
}
- protected BlockingQueue inQueue = new LinkedBlockingQueue<String>(10000);
+ protected BlockingQueue hosebirdQueue = new LinkedBlockingQueue<String>(1000);
- protected volatile Queue<StreamsDatum> providerQueue;
+ protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>(1000);
protected Hosts hosebirdHosts;
protected Authentication auth;
@@ -68,6 +61,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
protected ListeningExecutorService executor;
+ private DatumStatusCounter countersCurrent = new DatumStatusCounter();
+ private DatumStatusCounter countersTotal = new DatumStatusCounter();
+
private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
return new ThreadPoolExecutor(nThreads, nThreads,
5000L, TimeUnit.MILLISECONDS,
@@ -83,50 +79,43 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
this.config = config;
}
- public TwitterStreamProvider(Class klass) {
- Config config = StreamsConfigurator.config.getConfig("twitter");
- this.config = TwitterStreamConfigurator.detectConfiguration(config);
- this.klass = klass;
- providerQueue = new LinkedBlockingQueue<StreamsDatum>();
- }
-
- public TwitterStreamProvider(TwitterStreamConfiguration config, Class klass) {
- this.config = config;
- this.klass = klass;
- providerQueue = new LinkedBlockingQueue<StreamsDatum>();
-
- }
-
@Override
public void startStream() {
for (int i = 0; i < 5; i++) {
- executor.submit(new TwitterEventProcessor(inQueue, providerQueue, klass));
+ executor.submit(new TwitterEventProcessor(hosebirdQueue, providerQueue, String.class));
}
new Thread(new TwitterStreamProviderTask(this)).start();
+
}
@Override
public synchronized StreamsResultSet readCurrent() {
- Collection<StreamsDatum> currentIterator = Lists.newArrayList();
- Iterators.addAll(currentIterator, providerQueue.iterator());
- StreamsResultSet current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(currentIterator));
+ StreamsResultSet current;
- providerQueue.clear();
+ synchronized( TwitterStreamProvider.class ) {
+ current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(providerQueue));
+ current.setCounter(new DatumStatusCounter());
+ current.getCounter().add(countersCurrent);
+ countersTotal.add(countersCurrent);
+ countersCurrent = new DatumStatusCounter();
+ providerQueue.clear();
+ }
return current;
}
@Override
public StreamsResultSet readNew(BigInteger sequence) {
- return null;
+ throw new NotImplementedException();
}
@Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return null;
+ public StreamsResultSet readRange(DateTime start, DateTime end)
+ {
+ throw new NotImplementedException();
}
@Override
@@ -134,8 +123,6 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(5, 20));
- Preconditions.checkNotNull(this.klass);
-
Preconditions.checkNotNull(config.getEndpoint());
if(config.getEndpoint().equals("userstream") ) {
@@ -212,7 +199,8 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
.hosts(hosebirdHosts)
.endpoint(endpoint)
.authentication(auth)
- .processor(new StringDelimitedProcessor(inQueue))
+ .connectionTimeout(1200000)
+ .processor(new StringDelimitedProcessor(hosebirdQueue))
.build();
}
@@ -220,7 +208,12 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
@Override
public void cleanUp() {
for (int i = 0; i < 5; i++) {
- inQueue.add(TwitterEventProcessor.TERMINATE);
+ hosebirdQueue.add(TwitterEventProcessor.TERMINATE);
}
}
+
+ @Override
+ public DatumStatusCounter getDatumStatusCounter() {
+ return countersTotal;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index b9551ad..20ee951 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -2,10 +2,6 @@ package org.apache.streams.twitter.provider;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@@ -25,7 +21,9 @@ import twitter4j.json.DataObjectFactory;
import java.io.Serializable;
import java.math.BigInteger;
-import java.util.*;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
import java.util.concurrent.*;
/**
@@ -120,7 +118,8 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
@Override
public void startStream() {
- // no op
+ LOGGER.debug("{} startStream", STREAMS_ID);
+ throw new org.apache.commons.lang.NotImplementedException();
}
private void captureTimeline(long currentId) {
@@ -182,15 +181,20 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
}
public StreamsResultSet readCurrent() {
+ LOGGER.debug("{} readCurrent", STREAMS_ID);
Preconditions.checkArgument(ids.hasNext());
- LOGGER.info("readCurrent");
+ StreamsResultSet current;
+
+ synchronized( TwitterTimelineProvider.class ) {
+
+ while( ids.hasNext() ) {
+ Long currentId = ids.next();
+ LOGGER.info("Provider Task Starting: {}", currentId);
+ captureTimeline(currentId);
+ }
- while( ids.hasNext() ) {
- Long currentId = ids.next();
- LOGGER.info("Provider Task Starting: {}", currentId);
- captureTimeline(currentId);
}
LOGGER.info("Finished. Cleaning up...");
@@ -212,11 +216,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
public StreamsResultSet readRange(DateTime start, DateTime end) {
LOGGER.debug("{} readRange", STREAMS_ID);
- this.start = start;
- this.end = end;
- readCurrent();
- StreamsResultSet result = (StreamsResultSet)providerQueue.iterator();
- return result;
+ throw new NotImplementedException();
}
void shutdownAndAwaitTermination(ExecutorService pool) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index 5cf515c..101b47c 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -32,7 +32,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
private static final int START = 0;
private static final int END = 1;
- private static final int DEFAULT_TIMEOUT_MS = 30000;
+ private static final int DEFAULT_TIMEOUT_MS = 1000000;
private StreamsProvider provider;
private AtomicBoolean keepRunning;
[05/14] git commit: Fixed issue where exception terminated thread and
it wouldn't start again
Posted by sb...@apache.org.
Fixed issue where exception terminated thread and it wouldn't start again
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/f62afa57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/f62afa57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/f62afa57
Branch: refs/heads/master
Commit: f62afa57ce5ca19d639c84e89ea29f998a16fbe5
Parents: fb8f7ae
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 19:54:56 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 19:54:56 2014 -0400
----------------------------------------------------------------------
.../provider/SysomosHeartbeatStream.java | 45 +++++++++++---------
1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f62afa57/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
index c5145fb..9cd5898 100644
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
+++ b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
@@ -72,29 +72,36 @@ public class SysomosHeartbeatStream implements Runnable {
}
protected QueryResult executeAPIRequest() {
- BeatApi.BeatResponse response = this.client.createRequestBuilder()
- .setHeartBeatId(heartbeatId)
- .setOffset(0)
- .setReturnSetSize(maxApiBatch).execute();
-
- LOGGER.debug("Received {} results from API query", response.getCount());
+ BeatApi.BeatResponse response = null;
+ try {
+ response = this.client.createRequestBuilder()
+ .setHeartBeatId(heartbeatId)
+ .setOffset(0)
+ .setReturnSetSize(maxApiBatch).execute();
+
+ LOGGER.debug("Received {} results from API query", response.getCount());
+ } catch (Exception e) {
+ LOGGER.warn("Error querying Sysomos API", e);
+ }
String currentId = null;
boolean matched = false;
- for(BeatApi.BeatResponse.Beat beat : response.getBeat()) {
- String docId = beat.getDocid();
- //We get documents in descending time order. This will set the id to the latest document
- if(currentId == null) {
- currentId = docId;
- }
- //We only want to process documents that we know we have not seen before
- if(lastID != null && lastID.equals(docId)) {
- matched = true;
- break;
+ if(response != null) {
+ for (BeatApi.BeatResponse.Beat beat : response.getBeat()) {
+ String docId = beat.getDocid();
+ //We get documents in descending time order. This will set the id to the latest document
+ if (currentId == null) {
+ currentId = docId;
+ }
+ //We only want to process documents that we know we have not seen before
+ if (lastID != null && lastID.equals(docId)) {
+ matched = true;
+ break;
+ }
+ StreamsDatum item = new StreamsDatum(beat, docId);
+ item.getMetadata().put("heartbeat", this.heartbeatId);
+ this.provider.enqueueItem(item);
}
- StreamsDatum item = new StreamsDatum(beat, docId);
- item.getMetadata().put("heartbeat", this.heartbeatId);
- this.provider.enqueueItem(item);
}
return new QueryResult(matched, currentId);
}
[10/14] git commit: Merge branch 'streamstutorial' of
https://git-wip-us.apache.org/repos/asf/incubator-streams into
streamstutorial
Posted by sb...@apache.org.
Merge branch 'streamstutorial' of https://git-wip-us.apache.org/repos/asf/incubator-streams into streamstutorial
Conflicts:
streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/c59ab10e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/c59ab10e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/c59ab10e
Branch: refs/heads/master
Commit: c59ab10ed92c13d3b116c75f63a2fb921f0a43dc
Parents: 7e4e109 1cc9d0b
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 15:11:44 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 15:11:44 2014 -0500
----------------------------------------------------------------------
.../ElasticsearchConfigurator.java | 4 +-
.../ElasticsearchPersistWriter.java | 261 +++++++++++--------
.../ElasticsearchWriterConfiguration.json | 5 +-
.../provider/SysomosHeartbeatStream.java | 45 ++--
.../provider/TwitterTimelineProvider.java | 10 +-
.../local/builders/LocalStreamBuilder.java | 161 ++++++++----
.../local/tasks/StreamsPersistWriterTask.java | 6 +-
.../local/tasks/StreamsProcessorTask.java | 2 +-
8 files changed, 304 insertions(+), 190 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c59ab10e/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index e99c4ff,ce23197..97796e2
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@@ -373,26 -396,30 +397,29 @@@ public class ElasticsearchPersistWrite
}
public void add(IndexRequest indexRequest) {
- synchronized (this) {
- checkAndCreateBulkRequest();
- checkIndexImplications(indexRequest.index());
- bulkRequest.add(indexRequest);
- try {
- trackItemAndBytesWritten(indexRequest.source().length());
- } catch (NullPointerException x) {
- LOGGER.warn("NPE adding/sizing indexrequest");
- }
+ lock.writeLock().lock();
+ checkAndCreateBulkRequest();
+ checkIndexImplications(indexRequest.index());
+ bulkRequest.add(indexRequest);
+ try {
+ trackItemAndBytesWritten(indexRequest.source().length());
+ } catch (NullPointerException x) {
+ LOGGER.warn("NPE adding/sizing indexrequest");
+ } finally {
+ lock.writeLock().unlock();
}
-
}
- private void checkAndCreateBulkRequest()
+ private void trackItemAndBytesWritten(long sizeInBytes)
{
- // Synchronize to ensure that we don't lose any records
- synchronized (this)
- {
- if(bulkRequest == null)
- bulkRequest = this.manager.getClient().prepareBulk();
- }
+ currentItems++;
+ batchItemsSent++;
+ batchSizeInBytes += sizeInBytes;
+
+ // If our queue is larger than our flush threashold, then we should flush the queue.
+ if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+ (currentItems >= batchSize) )
+ flushInternal();
}
private void checkIndexImplications(String indexName)
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c59ab10e/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index 06ba7be,799abe7..3799737
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@@ -22,8 -22,12 +22,12 @@@ import twitter4j.conf.ConfigurationBuil
import twitter4j.json.DataObjectFactory;
import java.io.Serializable;
-import java.math.BigInteger;
+ import java.lang.Math;
+import java.math.BigInteger;
- import java.util.*;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
[11/14] git commit: fixed premature-timeout
Posted by sb...@apache.org.
fixed premature-timeout
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/87bd8a3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/87bd8a3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/87bd8a3f
Branch: refs/heads/master
Commit: 87bd8a3f874e711933b4e9c0a5e20518c9ff6f79
Parents: c59ab10
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 17:19:45 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 17:19:45 2014 -0500
----------------------------------------------------------------------
.../java/org/apache/streams/local/tasks/StreamsProviderTask.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/87bd8a3f/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index cd43d44..cc4844c 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -130,7 +130,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
zeros = 0;
}
flushResults(resultSet);
- if( zeros > (timeout / DEFAULT_SLEEP_TIME_MS))
+ // the way this works needs to change...
+ if( zeros > (timeout))
this.keepRunning.set(false);
Thread.sleep(DEFAULT_SLEEP_TIME_MS);
} catch (InterruptedException e) {
[14/14] git commit: Merge commit
'237f2a6ab13c68a61cf715004fe7b85db66237f6'
Posted by sb...@apache.org.
Merge commit '237f2a6ab13c68a61cf715004fe7b85db66237f6'
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e6830093
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e6830093
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e6830093
Branch: refs/heads/master
Commit: e68300936e100409ba4b518499eee66634d96245
Parents: c38eef6 237f2a6
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Tue May 13 10:21:52 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Tue May 13 10:21:52 2014 -0500
----------------------------------------------------------------------
.../streams-persist-elasticsearch/pom.xml | 1 -
.../ElasticsearchConfigurator.java | 11 +-
.../ElasticsearchPersistWriter.java | 297 +++++++++++--------
.../ElasticsearchWriterConfiguration.json | 15 +-
.../provider/SysomosHeartbeatStream.java | 45 +--
.../streams-provider-twitter/pom.xml | 13 +
.../twitter/processor/TwitterTypeConverter.java | 2 +-
.../provider/TwitterEventClassifier.java | 10 +-
.../twitter/provider/TwitterStreamProvider.java | 69 +++--
.../provider/TwitterTimelineProvider.java | 56 ++--
.../TwitterJsonActivitySerializer.java | 3 +-
.../TwitterJsonDeleteActivitySerializer.java | 3 +-
.../TwitterJsonRetweetActivitySerializer.java | 3 +-
.../TwitterJsonTweetActivitySerializer.java | 7 +-
...erJsonUserstreameventActivitySerializer.java | 92 ++++++
.../main/jsonschema/com/twitter/FriendList.json | 20 ++
.../jsonschema/com/twitter/UserstreamEvent.json | 47 +++
.../jackson/StreamsDateTimeDeserializer.java | 3 +-
.../jackson/StreamsDateTimeSerializer.java | 5 +-
.../jackson/StreamsPeriodDeserializer.java | 3 +-
.../jackson/StreamsPeriodSerializer.java | 3 +-
.../local/builders/LocalStreamBuilder.java | 161 ++++++----
.../local/tasks/StreamsPersistWriterTask.java | 6 +-
.../local/tasks/StreamsProcessorTask.java | 2 +-
.../local/tasks/StreamsProviderTask.java | 5 +-
25 files changed, 609 insertions(+), 273 deletions(-)
----------------------------------------------------------------------
[08/14] git commit: Merge remote-tracking branch 'origin/pr/7' into
streamstutorial
Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/pr/7' into streamstutorial
Conflicts:
streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/145ec847
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/145ec847
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/145ec847
Branch: refs/heads/master
Commit: 145ec8472c9cf12e8e376ce09596f354644e587b
Parents: 7293594 f62afa5
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 14:15:27 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 14:15:27 2014 -0500
----------------------------------------------------------------------
.../ElasticsearchConfigurator.java | 4 +-
.../ElasticsearchPersistWriter.java | 273 +++++++++----------
.../ElasticsearchWriterConfiguration.json | 5 +-
.../provider/SysomosHeartbeatStream.java | 45 +--
.../local/builders/LocalStreamBuilder.java | 161 +++++++----
.../local/tasks/StreamsPersistWriterTask.java | 6 +-
.../local/tasks/StreamsProcessorTask.java | 2 +-
7 files changed, 271 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/145ec847/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 9ac234c,8a343a6..e07e50f
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@@ -51,17 -51,12 +51,19 @@@ public class ElasticsearchConfigurator
String index = elasticsearch.getString("index");
String type = elasticsearch.getString("type");
+ Long maxMsBeforeFlush = elasticsearch.hasPath("MaxTimeBetweenFlushMs") ? elasticsearch.getLong("MaxTimeBetweenFlushMs") : null;
+ if( elasticsearch.hasPath("bulk"))
+ elasticsearchWriterConfiguration.setBulk(elasticsearch.getBoolean("bulk"));
+
+ if( elasticsearch.hasPath("batchSize"))
+ elasticsearchWriterConfiguration.setBatchSize(elasticsearch.getLong("batchSize"));
+
elasticsearchWriterConfiguration.setIndex(index);
elasticsearchWriterConfiguration.setType(type);
+ elasticsearchWriterConfiguration.setMaxTimeBetweenFlushMs(maxMsBeforeFlush);
+
return elasticsearchWriterConfiguration;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/145ec847/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 80d2775,5756e1c..ce23197
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@@ -49,8 -54,11 +54,12 @@@ public class ElasticsearchPersistWrite
private static final long WAITING_DOCS_LIMIT = 10000;
private static final int BYTES_IN_MB = 1024 * 1024;
private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+ private static final long DEFAULT_MAX_WAIT = 10000;
++ private static final int DEFAULT_BATCH_SIZE = 100;
private final List<String> affectedIndexes = new ArrayList<String>();
+ private final ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
private ObjectMapper mapper = new StreamsJacksonMapper();
private ElasticsearchClientManager manager;
@@@ -59,11 -67,11 +68,10 @@@
private String parentID = null;
private BulkRequestBuilder bulkRequest;
private OutputStreamWriter currentWriter = null;
- private int batchSize = 50;
- private int totalRecordsWritten = 0;
- private long maxMsBeforeFlush;
++ private int batchSize;
++ private long maxTimeBetweenFlushMs;
+ private boolean veryLargeBulk = false; // by default this setting is set to false
- private long batchSize;
- private boolean veryLargeBulk; // by default this setting is set to false
- private int totalRecordsWritten = 0;
-
protected Thread task;
protected volatile Queue<StreamsDatum> persistQueue;
@@@ -99,8 -106,7 +108,6 @@@
this.veryLargeBulk = veryLargeBulk;
}
- private final List<String> affectedIndexes = new ArrayList<String>();
--
public int getTotalOutstanding() {
return this.totalSent - (this.totalFailed + this.totalOk);
}
@@@ -151,6 -155,6 +156,14 @@@
this.flushThresholdSizeInBytes = sizeInBytes;
}
++ public long getMaxTimeBetweenFlushMs() {
++ return maxTimeBetweenFlushMs;
++ }
++
++ public void setMaxTimeBetweenFlushMs(long maxTimeBetweenFlushMs) {
++ this.maxTimeBetweenFlushMs = maxTimeBetweenFlushMs;
++ }
++
public boolean isConnected() {
return (client != null);
}
@@@ -252,12 -250,13 +259,6 @@@
}
@Override
-- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
- start();
- }
-
- @Override
- maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? DEFAULT_MAX_WAIT : config.getMaxTimeBetweenFlushMs();
- mapper = StreamsJacksonMapper.getInstance();
- start();
- }
-
- @Override
public DatumStatusCounter getDatumStatusCounter() {
DatumStatusCounter counters = new DatumStatusCounter();
counters.incrementAttempt(this.batchItemsSent);
@@@ -267,7 -266,17 +268,17 @@@
}
public void start() {
-
+ backgroundFlushTask.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ LOGGER.debug("Checking to see if data needs to be flushed");
+ long time = System.currentTimeMillis() - lastWrite.get();
- if (time > maxMsBeforeFlush && batchItemsSent > 0) {
++ if (time > maxTimeBetweenFlushMs && batchItemsSent > 0) {
+ LOGGER.debug("Background Flush task determined {} are waiting to be flushed. It has been {} since the last write to ES", batchItemsSent, time);
+ flushInternal();
+ }
+ }
- }, 0, maxMsBeforeFlush * 2, TimeUnit.MILLISECONDS);
++ }, 0, maxTimeBetweenFlushMs * 2, TimeUnit.MILLISECONDS);
manager = new ElasticsearchClientManager(config);
client = manager.getClient();
@@@ -383,70 -394,20 +396,62 @@@
}
public void add(IndexRequest indexRequest) {
- synchronized (this) {
- checkAndCreateBulkRequest();
- checkIndexImplications(indexRequest.index());
- bulkRequest.add(indexRequest);
- try {
- trackItemAndBytesWritten(indexRequest.source().length());
- } catch (NullPointerException x) {
- LOGGER.warn("NPE adding/sizing indexrequest");
- }
+ lock.writeLock().lock();
+ checkAndCreateBulkRequest();
+ checkIndexImplications(indexRequest.index());
+ bulkRequest.add(indexRequest);
+ try {
+ trackItemAndBytesWritten(indexRequest.source().length());
+ } catch (NullPointerException x) {
+ LOGGER.warn("NPE adding/sizing indexrequest");
+ } finally {
+ lock.writeLock().unlock();
}
+
}
+ private void trackItemAndBytesWritten(long sizeInBytes)
+ {
+ currentItems++;
+ batchItemsSent++;
+ batchSizeInBytes += sizeInBytes;
+
+ // If our queue is larger than our flush threashold, then we should flush the queue.
+ if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+ (currentItems >= batchSize) )
+ flushInternal();
+ }
+
- private void checkAndCreateBulkRequest()
- {
- // Synchronize to ensure that we don't lose any records
- synchronized (this)
- {
- if(bulkRequest == null)
- bulkRequest = this.manager.getClient().prepareBulk();
- }
- }
-
+ private void checkIndexImplications(String indexName)
+ {
+
+ // check to see if we have seen this index before.
+ if(this.affectedIndexes.contains(indexName))
+ return;
+
+ // we haven't log this index.
+ this.affectedIndexes.add(indexName);
+
+ // Check to see if we are in 'veryLargeBulk' mode
+ // if we aren't, exit early
+ if(!this.veryLargeBulk)
+ return;
+
+
+ // They are in 'very large bulk' mode we want to turn off refreshing the index.
+
+ // Create a request then add the setting to tell it to stop refreshing the interval
+ UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
+ updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
+
+ // submit to ElasticSearch
+ this.manager.getClient()
+ .admin()
+ .indices()
+ .updateSettings(updateSettingsRequest)
+ .actionGet();
+ }
+
public void createIndexIfMissing(String indexName) {
if (!this.manager.getClient()
.admin()
@@@ -512,15 -473,15 +517,24 @@@
return toReturn;
}
+ @Override
+ public void prepare(Object configurationObject) {
+ mapper = StreamsJacksonMapper.getInstance();
- veryLargeBulk = this.config.getBulk();
- batchSize = this.config.getBatchSize();
++ veryLargeBulk = config.getBulk() == null ? Boolean.FALSE : config.getBulk();
++ batchSize = config.getBatchSize() == null ? DEFAULT_BATCH_SIZE : (int)(config.getBatchSize().longValue());
++ maxTimeBetweenFlushMs = config.getMaxTimeBetweenFlushMs() == null ? DEFAULT_MAX_WAIT : config.getMaxTimeBetweenFlushMs().longValue();
+ start();
+ }
+
+ /**
+ * This method is to ONLY be called by flushInternal otherwise the counts will be off.
+ * @param bulkRequest
+ * @param thisSent
+ * @param thisSizeInBytes
+ */
private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+ final Object messenger = new Object();
+ LOGGER.debug("Attempting to write {} items to ES", thisSent);
bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkItemResponses) {
@@@ -558,18 -521,16 +574,9 @@@
e.printStackTrace();
}
});
-
- this.notify();
}
-- private void trackItemAndBytesWritten(long sizeInBytes) {
-- batchItemsSent++;
-- batchSizeInBytes += sizeInBytes;
-- // If our queue is larger than our flush threashold, then we should flush the queue.
-- if (batchSizeInBytes > flushThresholdSizeInBytes)
-- flushInternal();
-- }
private void checkAndCreateBulkRequest() {
// Synchronize to ensure that we don't lose any records
@@@ -579,32 -543,32 +589,5 @@@
}
}
-- private void checkIndexImplications(String indexName) {
--
-- // check to see if we have seen this index before.
-- if (this.affectedIndexes.contains(indexName))
-- return;
--
-- // we haven't log this index.
-- this.affectedIndexes.add(indexName);
--
-- // Check to see if we are in 'veryLargeBulk' mode
-- // if we aren't, exit early
-- if (!this.veryLargeBulk)
-- return;
--
--
-- // They are in 'very large bulk' mode we want to turn off refreshing the index.
--
-- // Create a request then add the setting to tell it to stop refreshing the interval
-- UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
-- updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
--
-- // submit to ElasticSearch
-- this.manager.getClient()
-- .admin()
-- .indices()
-- .updateSettings(updateSettingsRequest)
-- .actionGet();
-- }
}
++
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/145ec847/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 5e169b7,c56aa82..b107be6
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@@ -14,15 -14,9 +14,18 @@@
"type": "string",
"description": "Type to write as"
},
- "MaxTimeBetweenFlushMs": {
- "type": "String",
- "format": "utc-millisec"
+ "bulk": {
+ "type": "boolean",
+ "description": "Index in large or small batches",
+ "default": "false"
+ },
+ "batchSize": {
+ "type": "integer",
+ "description": "Item Count before flush",
+ "default": 100
- }
++ },
++ "maxTimeBetweenFlushMs": {
++ "type": "integer"
+ }
}
}
[03/14] git commit: Updated termination process
Posted by sb...@apache.org.
Updated termination process
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/6934c7a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/6934c7a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/6934c7a0
Branch: refs/heads/master
Commit: 6934c7a0d4c479392c6c147d00af8cd9ed7b3c44
Parents: c837a2c
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 18:45:30 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 18:45:30 2014 -0400
----------------------------------------------------------------------
.../local/builders/LocalStreamBuilder.java | 161 ++++++++++++-------
.../local/tasks/StreamsPersistWriterTask.java | 6 +-
.../local/tasks/StreamsProcessorTask.java | 2 +-
3 files changed, 110 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6934c7a0/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 8e688ba..d313b3f 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -1,5 +1,6 @@
package org.apache.streams.local.builders;
+import org.apache.log4j.spi.LoggerFactory;
import org.apache.streams.core.*;
import org.apache.streams.local.tasks.LocalStreamProcessMonitorThread;
import org.apache.streams.local.tasks.StatusCounterMonitorThread;
@@ -7,6 +8,7 @@ import org.apache.streams.local.tasks.StreamsProviderTask;
import org.apache.streams.local.tasks.StreamsTask;
import org.apache.streams.util.SerializationUtil;
import org.joda.time.DateTime;
+import org.slf4j.Logger;
import java.math.BigInteger;
import java.util.*;
@@ -22,6 +24,8 @@ import java.util.concurrent.TimeUnit;
*/
public class LocalStreamBuilder implements StreamBuilder {
+ private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(LocalStreamBuilder.class);
+
public static final String TIMEOUT_KEY = "TIMEOUT";
private Map<String, StreamComponent> providers;
private Map<String, StreamComponent> components;
@@ -32,6 +36,7 @@ public class LocalStreamBuilder implements StreamBuilder {
private int totalTasks;
private int monitorTasks;
private LocalStreamProcessMonitorThread monitorThread;
+ private Map<String, List<StreamsTask>> tasks;
/**
*
@@ -139,39 +144,18 @@ public class LocalStreamBuilder implements StreamBuilder {
*/
@Override
public void start() {
+ attachShutdownHandler();
boolean isRunning = true;
this.executor = Executors.newFixedThreadPool(this.totalTasks);
this.monitor = Executors.newFixedThreadPool(this.monitorTasks+1);
Map<String, StreamsProviderTask> provTasks = new HashMap<String, StreamsProviderTask>();
- Map<String, List<StreamsTask>> streamsTasks = new HashMap<String, List<StreamsTask>>();
+ tasks = new HashMap<String, List<StreamsTask>>();
try {
monitorThread = new LocalStreamProcessMonitorThread(executor, 10);
this.monitor.submit(monitorThread);
- for(StreamComponent comp : this.components.values()) {
- int tasks = comp.getNumTasks();
- List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
- for(int i=0; i < tasks; ++i) {
- StreamsTask task = comp.createConnectedTask(getTimeout());
- task.setStreamConfig(this.streamConfig);
- this.executor.submit(task);
- compTasks.add(task);
- if( comp.isOperationCountable() ) {
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) comp.getOperation(), 10));
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
- }
- }
- streamsTasks.put(comp.getId(), compTasks);
- }
- for(StreamComponent prov : this.providers.values()) {
- StreamsTask task = prov.createConnectedTask(getTimeout());
- task.setStreamConfig(this.streamConfig);
- this.executor.submit(task);
- provTasks.put(prov.getId(), (StreamsProviderTask) task);
- if( prov.isOperationCountable() ) {
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) prov.getOperation(), 10));
- this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
- }
- }
+ setupComponentTasks(tasks);
+ setupProviderTasks(provTasks);
+ LOGGER.info("Started stream with {} components", tasks.size());
while(isRunning) {
isRunning = false;
for(StreamsProviderTask task : provTasks.values()) {
@@ -184,44 +168,98 @@ public class LocalStreamBuilder implements StreamBuilder {
Thread.sleep(3000);
}
}
- monitorThread.shutdown();
- this.executor.shutdown();
- //complete stream shut down gracfully
- for(StreamComponent prov : this.providers.values()) {
- shutDownTask(prov, streamsTasks);
+ LOGGER.debug("Components are no longer running or timed out due to completion");
+ shutdown(tasks);
+ } catch (InterruptedException e){
+ forceShutdown(tasks);
+ }
+
+ }
+
+ private void attachShutdownHandler() {
+ final LocalStreamBuilder self = this;
+ LOGGER.debug("Attaching shutdown handler");
+ Runtime.getRuntime().addShutdownHook(new Thread(){
+ @Override
+ public void run() {
+ LOGGER.debug("Shutdown hook received. Beginning shutdown");
+ self.stop();
+ }
+ });
+ }
+
+ protected void forceShutdown(Map<String, List<StreamsTask>> streamsTasks) {
+ LOGGER.debug("Shutdown failed. Forcing shutdown");
+ //give the stream 30secs to try to shutdown gracefully, then force shutdown otherwise
+ for(List<StreamsTask> tasks : streamsTasks.values()) {
+ for(StreamsTask task : tasks) {
+ task.stopTask();
}
- //need to make this configurable
- if(!this.executor.awaitTermination(10, TimeUnit.SECONDS)) { // all threads should have terminated already.
+ }
+ this.executor.shutdown();
+ this.monitor.shutdown();
+ try {
+ if(!this.executor.awaitTermination(3, TimeUnit.SECONDS)){
this.executor.shutdownNow();
- this.executor.awaitTermination(10, TimeUnit.SECONDS);
}
- if(!this.monitor.awaitTermination(5, TimeUnit.SECONDS)) { // all threads should have terminated already.
+ if(!this.monitor.awaitTermination(3, TimeUnit.SECONDS)){
this.monitor.shutdownNow();
- this.monitor.awaitTermination(5, TimeUnit.SECONDS);
}
- } catch (InterruptedException e){
- //give the stream 30secs to try to shutdown gracefully, then force shutdown otherwise
- for(List<StreamsTask> tasks : streamsTasks.values()) {
- for(StreamsTask task : tasks) {
- task.stopTask();
- }
+ }catch (InterruptedException ie) {
+ this.executor.shutdownNow();
+ this.monitor.shutdownNow();
+ throw new RuntimeException(ie);
+ }
+ }
+
+ protected void shutdown(Map<String, List<StreamsTask>> streamsTasks) throws InterruptedException {
+ LOGGER.info("Attempting to shutdown tasks");
+ monitorThread.shutdown();
+ this.executor.shutdown();
+ //complete stream shut down gracfully
+ for(StreamComponent prov : this.providers.values()) {
+ shutDownTask(prov, streamsTasks);
+ }
+ //need to make this configurable
+ if(!this.executor.awaitTermination(10, TimeUnit.SECONDS)) { // all threads should have terminated already.
+ this.executor.shutdownNow();
+ this.executor.awaitTermination(10, TimeUnit.SECONDS);
+ }
+ if(!this.monitor.awaitTermination(5, TimeUnit.SECONDS)) { // all threads should have terminated already.
+ this.monitor.shutdownNow();
+ this.monitor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ }
+
+ protected void setupProviderTasks(Map<String, StreamsProviderTask> provTasks) {
+ for(StreamComponent prov : this.providers.values()) {
+ StreamsTask task = prov.createConnectedTask(getTimeout());
+ task.setStreamConfig(this.streamConfig);
+ this.executor.submit(task);
+ provTasks.put(prov.getId(), (StreamsProviderTask) task);
+ if( prov.isOperationCountable() ) {
+ this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) prov.getOperation(), 10));
+ this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
}
- this.executor.shutdown();
- this.monitor.shutdown();
- try {
- if(!this.executor.awaitTermination(3, TimeUnit.SECONDS)){
- this.executor.shutdownNow();
- }
- if(!this.monitor.awaitTermination(3, TimeUnit.SECONDS)){
- this.monitor.shutdownNow();
+ }
+ }
+
+ protected void setupComponentTasks(Map<String, List<StreamsTask>> streamsTasks) {
+ for(StreamComponent comp : this.components.values()) {
+ int tasks = comp.getNumTasks();
+ List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
+ for(int i=0; i < tasks; ++i) {
+ StreamsTask task = comp.createConnectedTask(getTimeout());
+ task.setStreamConfig(this.streamConfig);
+ this.executor.submit(task);
+ compTasks.add(task);
+ if( comp.isOperationCountable() ) {
+ this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) comp.getOperation(), 10));
+ this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
}
- }catch (InterruptedException ie) {
- this.executor.shutdownNow();
- this.monitor.shutdownNow();
- throw new RuntimeException(ie);
}
+ streamsTasks.put(comp.getId(), compTasks);
}
-
}
/**
@@ -249,8 +287,13 @@ public class LocalStreamBuilder implements StreamBuilder {
task.stopTask();
}
for(StreamsTask task : tasks) {
- while(task.isRunning()) {
+ int count = 0;
+ while(count < 20 && task.isRunning()) {
Thread.sleep(500);
+ count++;
+ }
+ if(task.isRunning()) {
+ LOGGER.warn("Task {} failed to terminate in allotted timeframe", task.toString());
}
}
}
@@ -268,7 +311,11 @@ public class LocalStreamBuilder implements StreamBuilder {
*/
@Override
public void stop() {
-
+ try {
+ shutdown(tasks);
+ } catch (Exception e) {
+ forceShutdown(tasks);
+ }
}
private void connectToOtherComponents(String[] conntectToIds, StreamComponent toBeConnected) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6934c7a0/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index 1eac1d9..8146bdd 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -72,12 +72,13 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
try {
this.writer.prepare(this.streamConfig);
StreamsDatum datum = this.inQueue.poll();
- while(datum != null || this.keepRunning.get()) {
+ while(this.keepRunning.get()) {
if(datum != null) {
try {
this.writer.write(datum);
statusCounter.incrementStatus(DatumStatus.SUCCESS);
} catch (Exception e) {
+ LOGGER.error("Error writing to persist writer {}", this.writer.getClass().getSimpleName(), e);
this.keepRunning.set(false);
statusCounter.incrementStatus(DatumStatus.FAIL);
}
@@ -86,12 +87,15 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
try {
Thread.sleep(this.sleepTime);
} catch (InterruptedException e) {
+ LOGGER.warn("Thread interrupted in Writer task for {}",this.writer.getClass().getSimpleName(), e);
this.keepRunning.set(false);
}
}
datum = this.inQueue.poll();
}
+ } catch(Exception e) {
+ LOGGER.error("Failed to execute Persist Writer {}",this.writer.getClass().getSimpleName(), e);
} finally {
this.writer.cleanUp();
this.isRunning.set(false);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6934c7a0/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index d1ac905..d4c7a16 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -67,7 +67,7 @@ public class StreamsProcessorTask extends BaseStreamsTask {
try {
this.processor.prepare(this.streamConfig);
StreamsDatum datum = this.inQueue.poll();
- while(datum != null || this.keepRunning.get()) {
+ while(this.keepRunning.get()) {
if(datum != null) {
List<StreamsDatum> output = this.processor.process(datum);
if(output != null) {
[07/14] git commit: fixes to get streams-examples working
Posted by sb...@apache.org.
fixes to get streams-examples working
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7e4e1095
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7e4e1095
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7e4e1095
Branch: refs/heads/master
Commit: 7e4e1095f09321210d6ce3acb56f4041680f2d6d
Parents: 7293594
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 13:41:28 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 13:41:28 2014 -0500
----------------------------------------------------------------------
.../ElasticsearchPersistWriter.java | 80 ++---------------
.../streams-provider-twitter/pom.xml | 13 +++
.../provider/TwitterEventClassifier.java | 10 ++-
.../twitter/provider/TwitterStreamProvider.java | 7 +-
.../provider/TwitterTimelineProvider.java | 7 +-
.../TwitterJsonActivitySerializer.java | 3 +-
.../TwitterJsonDeleteActivitySerializer.java | 3 +-
.../TwitterJsonRetweetActivitySerializer.java | 3 +-
.../TwitterJsonTweetActivitySerializer.java | 7 +-
...erJsonUserstreameventActivitySerializer.java | 92 ++++++++++++++++++++
.../main/jsonschema/com/twitter/FriendList.json | 20 +++++
.../jsonschema/com/twitter/UserstreamEvent.json | 47 ++++++++++
.../jackson/StreamsDateTimeDeserializer.java | 3 +-
.../jackson/StreamsDateTimeSerializer.java | 5 +-
.../jackson/StreamsPeriodDeserializer.java | 3 +-
.../jackson/StreamsPeriodSerializer.java | 3 +-
16 files changed, 211 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 80d2775..e99c4ff 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -47,10 +47,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
private static final Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
private static final long WAITING_DOCS_LIMIT = 10000;
- private static final int BYTES_IN_MB = 1024 * 1024;
- private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
-
- private final List<String> affectedIndexes = new ArrayList<String>();
+ private static final int BYTES_IN_MB = 1024*1024;
+ private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
private ObjectMapper mapper = new StreamsJacksonMapper();
private ElasticsearchClientManager manager;
@@ -62,8 +60,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private long batchSize;
private boolean veryLargeBulk; // by default this setting is set to false
- private int totalRecordsWritten = 0;
-
+
protected Thread task;
protected volatile Queue<StreamsDatum> persistQueue;
@@ -79,8 +76,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private volatile long totalSizeInBytes = 0;
private volatile long batchSizeInBytes = 0;
private volatile int batchItemsSent = 0;
- private volatile int totalByteCount = 0;
- private volatile int byteCount = 0;
+ private volatile int totalByteCount = 0;
+ private volatile int byteCount = 0;
public ElasticsearchPersistWriter() {
Config config = StreamsConfigurator.config.getConfig("elasticsearch");
@@ -121,8 +118,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
return totalOk;
}
- private ObjectMapper mapper = new StreamsJacksonMapper();
-
public int getTotalFailed() {
return totalFailed;
}
@@ -155,13 +150,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
return (client != null);
}
- private ElasticsearchWriterConfiguration config;
-
- private static final int BYTES_IN_MB = 1024*1024;
- private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
- private volatile int totalByteCount = 0;
- private volatile int byteCount = 0;
-
@Override
public void write(StreamsDatum streamsDatum) {
@@ -254,6 +242,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
@Override
public void prepare(Object configurationObject) {
mapper = StreamsJacksonMapper.getInstance();
+ veryLargeBulk = this.config.getBulk();
+ batchSize = this.config.getBatchSize();
start();
}
@@ -395,18 +385,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
}
- private void trackItemAndBytesWritten(long sizeInBytes)
- {
- currentItems++;
- batchItemsSent++;
- batchSizeInBytes += sizeInBytes;
-
- // If our queue is larger than our flush threashold, then we should flush the queue.
- if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
- (currentItems >= batchSize) )
- flushInternal();
- }
-
private void checkAndCreateBulkRequest()
{
// Synchronize to ensure that we don't lose any records
@@ -512,14 +490,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
return toReturn;
}
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
- veryLargeBulk = this.config.getBulk();
- batchSize = this.config.getBatchSize();
- start();
- }
-
private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
@Override
@@ -571,40 +541,4 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
flushInternal();
}
- private void checkAndCreateBulkRequest() {
- // Synchronize to ensure that we don't lose any records
- synchronized (this) {
- if (bulkRequest == null)
- bulkRequest = this.manager.getClient().prepareBulk();
- }
- }
-
- private void checkIndexImplications(String indexName) {
-
- // check to see if we have seen this index before.
- if (this.affectedIndexes.contains(indexName))
- return;
-
- // we haven't log this index.
- this.affectedIndexes.add(indexName);
-
- // Check to see if we are in 'veryLargeBulk' mode
- // if we aren't, exit early
- if (!this.veryLargeBulk)
- return;
-
-
- // They are in 'very large bulk' mode we want to turn off refreshing the index.
-
- // Create a request then add the setting to tell it to stop refreshing the interval
- UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
- updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
-
- // submit to ElasticSearch
- this.manager.getClient()
- .admin()
- .indices()
- .updateSettings(updateSettingsRequest)
- .actionGet();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index 8a41ca5..c104810 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -49,6 +49,11 @@
<artifactId>streams-config</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.streams</groupId>
+ <artifactId>streams-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
@@ -70,6 +75,12 @@
<artifactId>twitter4j-core</artifactId>
<version>[4.0,)</version>
</dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
@@ -116,6 +127,8 @@
<sourcePath>src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/Delete.json</sourcePath>
+ <sourcePath>src/main/jsonschema/com/twitter/UserstreamEvent.json</sourcePath>
+ <sourcePath>src/main/jsonschema/com/twitter/FriendList.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/Retweet.json</sourcePath>
<sourcePath>src/main/jsonschema/com/twitter/tweet.json</sourcePath>
</sourcePaths>
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
index b577e42..3a8bf6d 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
@@ -2,11 +2,8 @@ package org.apache.streams.twitter.provider;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Preconditions;
-import com.jayway.jsonassert.JsonAssert;
import org.apache.commons.lang.StringUtils;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
+import org.apache.streams.twitter.pojo.*;
import org.apache.streams.twitter.serializer.StreamsTwitterMapper;
import java.io.IOException;
@@ -47,6 +44,11 @@ public class TwitterEventClassifier {
return Retweet.class;
else if( objectNode.findValue("delete") != null )
return Delete.class;
+ else if( objectNode.findValue("friends") != null ||
+ objectNode.findValue("friends_str") != null )
+ return FriendList.class;
+ else if( objectNode.findValue("target_object") != null )
+ return UserstreamEvent.class;
else
return Tweet.class;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
index ba88803..fb160ef 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
@@ -50,9 +50,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable, Dat
this.config = config;
}
- protected BlockingQueue hosebirdQueue = new LinkedBlockingQueue<String>(1000);
+ protected BlockingQueue<String> hosebirdQueue;
- protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>(1000);
+ protected volatile Queue<StreamsDatum> providerQueue;
protected Hosts hosebirdHosts;
protected Authentication auth;
@@ -194,6 +194,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable, Dat
LOGGER.debug("host={}\tendpoint={}\taut={}", new Object[] {hosebirdHosts,endpoint,auth});
+ hosebirdQueue = new LinkedBlockingQueue<String>(1000);
+ providerQueue = new LinkedBlockingQueue<StreamsDatum>(1000);
+
client = new ClientBuilder()
.name("apache/streams/streams-contrib/streams-provider-twitter")
.hosts(hosebirdHosts)
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index db1ec76..06ba7be 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -23,9 +23,7 @@ import twitter4j.json.DataObjectFactory;
import java.io.Serializable;
import java.math.BigInteger;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Queue;
+import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
@@ -260,7 +258,8 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
.setAsyncNumThreads(3)
.setRestBaseURL(baseUrl)
.setIncludeMyRetweetEnabled(Boolean.TRUE)
- .setIncludeRTsEnabled(Boolean.TRUE)
+ // not sure where this method went...
+ //.setIncludeRTsEnabled(Boolean.TRUE)
.setPrettyDebugEnabled(Boolean.TRUE);
return new TwitterFactory(builder.build()).getInstance();
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index bfceae0..0ab3448 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -31,13 +31,14 @@ import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import java.io.IOException;
+import java.io.Serializable;
import java.util.List;
import java.util.Map;
/**
* Created by sblackmon on 3/26/14.
*/
-public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
+public class TwitterJsonActivitySerializer implements ActivitySerializer<String>, Serializable
{
public TwitterJsonActivitySerializer() {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
index 40be0f6..4a0f348 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
@@ -13,6 +13,7 @@ import org.apache.streams.pojo.json.Actor;
import org.apache.streams.twitter.pojo.Delete;
import org.apache.streams.twitter.pojo.Tweet;
+import java.io.Serializable;
import java.util.List;
import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerializer.*;
@@ -24,7 +25,7 @@ import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerialize
* Time: 9:24 AM
* To change this template use File | Settings | File Templates.
*/
-public class TwitterJsonDeleteActivitySerializer implements ActivitySerializer<String> {
+public class TwitterJsonDeleteActivitySerializer implements ActivitySerializer<String>, Serializable {
@Override
public String serializationFormat() {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
index 51e39b1..f8ade3f 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
@@ -17,6 +17,7 @@ import org.apache.streams.twitter.pojo.Tweet;
import org.apache.streams.twitter.pojo.User;
import java.io.IOException;
+import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -31,7 +32,7 @@ import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
* Time: 9:24 AM
* To change this template use File | Settings | File Templates.
*/
-public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<String> {
+public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<String>, Serializable {
public TwitterJsonRetweetActivitySerializer() {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
index b141482..1d00b7a 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
@@ -18,6 +18,7 @@ import org.apache.streams.twitter.pojo.Tweet;
import org.apache.streams.twitter.pojo.User;
import java.io.IOException;
+import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -32,11 +33,7 @@ import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
* Time: 9:24 AM
* To change this template use File | Settings | File Templates.
*/
-public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<String> {
-
- public TwitterJsonTweetActivitySerializer() {
-
- }
+public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<String>, Serializable {
@Override
public String serializationFormat() {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java
new file mode 100644
index 0000000..c0768e3
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java
@@ -0,0 +1,92 @@
+package org.apache.streams.twitter.serializer;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Strings;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.streams.data.ActivitySerializer;
+import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.twitter.pojo.Delete;
+import org.apache.streams.twitter.pojo.Tweet;
+import org.apache.streams.twitter.pojo.UserstreamEvent;
+
+import java.util.List;
+
+import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerializer.*;
+
+/**
+* Created with IntelliJ IDEA.
+* User: mdelaet
+* Date: 9/30/13
+* Time: 9:24 AM
+* To change this template use File | Settings | File Templates.
+*/
+public class TwitterJsonUserstreameventActivitySerializer implements ActivitySerializer<String> {
+
+ @Override
+ public String serializationFormat() {
+ return null;
+ }
+
+ @Override
+ public String serialize(Activity deserialized) throws ActivitySerializerException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Activity deserialize(String serialized) throws ActivitySerializerException {
+ return null;
+ }
+
+ @Override
+ public List<Activity> deserializeAll(List<String> serializedList) {
+ return null;
+ }
+
+ public Activity convert(ObjectNode item) throws ActivitySerializerException {
+
+ ObjectMapper mapper = StreamsTwitterMapper.getInstance();
+ UserstreamEvent event = null;
+ try {
+ event = mapper.treeToValue(item, UserstreamEvent.class);
+ } catch (JsonProcessingException e) {
+ e.printStackTrace();
+ }
+
+ Activity activity = new Activity();
+ activity.setActor(buildActor(event));
+ activity.setVerb(detectVerb(event));
+ activity.setObject(buildActivityObject(event));
+ activity.setId(TwitterJsonActivitySerializer.formatId(activity.getVerb()));
+ if(Strings.isNullOrEmpty(activity.getId()))
+ throw new ActivitySerializerException("Unable to determine activity id");
+ activity.setProvider(getProvider());
+ return activity;
+ }
+
+ public Actor buildActor(UserstreamEvent event) {
+ Actor actor = new Actor();
+ //actor.setId(formatId(delete.getDelete().getStatus().getUserIdStr()));
+ return actor;
+ }
+
+ public ActivityObject buildActivityObject(UserstreamEvent event) {
+ ActivityObject actObj = new ActivityObject();
+ //actObj.setId(formatId(delete.getDelete().getStatus().getIdStr()));
+ //actObj.setObjectType("tweet");
+ return actObj;
+ }
+
+ public String detectVerb(UserstreamEvent event) {
+ return null;
+ }
+
+ public ActivityObject buildTarget(UserstreamEvent event) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
new file mode 100644
index 0000000..5dd7687
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
@@ -0,0 +1,20 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.FriendList",
+ "properties": {
+ "friends": {
+ "type": "array",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "friends_str": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
new file mode 100644
index 0000000..07b5883
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
@@ -0,0 +1,47 @@
+{
+ "type": "object",
+ "$schema": "http://json-schema.org/draft-03/schema",
+ "id": "#",
+ "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
+ "properties": {
+ "created_at": {
+ "type": "string",
+ "format" : "date-time"
+ },
+ "event_type": {
+ "type": "string",
+ "enum" : [
+ "access_revoked",
+ "block",
+ "unblock",
+ "favorite",
+ "unfavorite",
+ "follow",
+ "unfollow",
+ "list_created",
+ "list_destroyed",
+ "list_updated",
+ "list_member_added",
+ "list_member_removed",
+ "list_user_subscribed",
+ "list_user_unsubscribed",
+ "user_update"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "target": {
+ "type": "string",
+ "items": {
+ "type": "integer"
+ }
+ },
+ "target_object": {
+ "type": "object"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
index 3286e74..28f297c 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
@@ -7,11 +7,12 @@ import org.apache.streams.data.util.RFC3339Utils;
import org.joda.time.DateTime;
import java.io.IOException;
+import java.io.Serializable;
/**
* Created by sblackmon on 3/27/14.
*/
-public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> {
+public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> implements Serializable {
protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass) {
super(dateTimeClass);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
index 26bc157..89f9100 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
@@ -11,11 +11,14 @@ import org.joda.time.format.DateTimeFormatter;
import org.joda.time.format.ISODateTimeFormat;
import java.io.IOException;
+import java.io.Serializable;
/**
* Created by sblackmon on 3/27/14.
*/
-public class StreamsDateTimeSerializer extends StdSerializer<DateTime> {
+public class StreamsDateTimeSerializer extends StdSerializer<DateTime> implements Serializable {
+
+
protected StreamsDateTimeSerializer(Class<DateTime> dateTimeClass) {
super(dateTimeClass);
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
index ff765f6..9402878 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
@@ -7,8 +7,9 @@ import org.apache.streams.data.util.RFC3339Utils;
import org.joda.time.Period;
import java.io.IOException;
+import java.io.Serializable;
-public class StreamsPeriodDeserializer extends StdDeserializer<Period>
+public class StreamsPeriodDeserializer extends StdDeserializer<Period> implements Serializable
{
protected StreamsPeriodDeserializer(Class<Period> dateTimeClass) {
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
index 614cbdd..ef28c8e 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
@@ -8,8 +8,9 @@ import org.joda.time.DateTime;
import org.joda.time.Period;
import java.io.IOException;
+import java.io.Serializable;
-public class StreamsPeriodSerializer extends StdSerializer<Period>
+public class StreamsPeriodSerializer extends StdSerializer<Period> implements Serializable
{
protected StreamsPeriodSerializer(Class<Period> dateTimeClass) {
super(dateTimeClass);
[06/14] git commit: merging fixes and addressing serialization
problems to get twitter streams-examples working.
Posted by sb...@apache.org.
merging fixes and addressing serialization problems to get twitter streams-examples working.
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7293594e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7293594e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7293594e
Branch: refs/heads/master
Commit: 7293594eac98535e4b2206dca68da9b73e6cdf24
Parents: ae27541 522096e
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 12:56:36 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 12:56:36 2014 -0500
----------------------------------------------------------------------
.../streams-persist-elasticsearch/pom.xml | 1 -
.../ElasticsearchConfigurator.java | 7 ++
.../ElasticsearchPersistWriter.java | 80 +++++++++++++++++++-
.../ElasticsearchWriterConfiguration.json | 10 +++
.../twitter/processor/TwitterTypeConverter.java | 2 +-
.../twitter/provider/TwitterStreamProvider.java | 68 ++++++++---------
.../provider/TwitterTimelineProvider.java | 50 ++++++++----
.../local/tasks/StreamsProviderTask.java | 2 +-
8 files changed, 161 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ff7e0f5,405d1b8..80d2775
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@@ -59,96 -50,68 +59,109 @@@ public class ElasticsearchPersistWrite
private String parentID = null;
private BulkRequestBuilder bulkRequest;
private OutputStreamWriter currentWriter = null;
- private int batchSize = 50;
- private int totalRecordsWritten = 0;
- private boolean veryLargeBulk = false; // by default this setting is set to false
+ private long batchSize;
+ private boolean veryLargeBulk; // by default this setting is set to false
++ private int totalRecordsWritten = 0;
++
+ protected Thread task;
- private final static Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
- private static final long WAITING_DOCS_LIMIT = 10000;
-
- public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
+ protected volatile Queue<StreamsDatum> persistQueue;
+ private volatile int currentItems = 0;
private volatile int totalSent = 0;
private volatile int totalSeconds = 0;
private volatile int totalAttempted = 0;
private volatile int totalOk = 0;
private volatile int totalFailed = 0;
private volatile int totalBatchCount = 0;
+ private volatile int totalRecordsWritten = 0;
private volatile long totalSizeInBytes = 0;
-
private volatile long batchSizeInBytes = 0;
private volatile int batchItemsSent = 0;
+ private volatile int totalByteCount = 0;
+ private volatile int byteCount = 0;
+
+ public ElasticsearchPersistWriter() {
+ Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+ this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+ }
+
+ public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
+ this.config = config;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public void setVeryLargeBulk(boolean veryLargeBulk) {
+ this.veryLargeBulk = veryLargeBulk;
+ }
+ private final List<String> affectedIndexes = new ArrayList<String>();
- public int getTotalOutstanding() { return this.totalSent - (this.totalFailed + this.totalOk); }
- public long getFlushThresholdSizeInBytes() { return flushThresholdSizeInBytes; }
- public int getTotalSent() { return totalSent; }
- public int getTotalSeconds() { return totalSeconds; }
- public int getTotalOk() { return totalOk; }
- public int getTotalFailed() { return totalFailed; }
- public int getTotalBatchCount() { return totalBatchCount; }
- public long getTotalSizeInBytes() { return totalSizeInBytes; }
- public long getBatchSizeInBytes() { return batchSizeInBytes; }
- public int getBatchItemsSent() { return batchItemsSent; }
- public List<String> getAffectedIndexes() { return this.affectedIndexes; }
+ public int getTotalOutstanding() {
+ return this.totalSent - (this.totalFailed + this.totalOk);
+ }
- public void setFlushThresholdSizeInBytes(long sizeInBytes) { this.flushThresholdSizeInBytes = sizeInBytes; }
+ public long getFlushThresholdSizeInBytes() {
+ return flushThresholdSizeInBytes;
+ }
- Thread task;
+ public int getTotalSent() {
+ return totalSent;
+ }
- protected volatile Queue<StreamsDatum> persistQueue;
+ public int getTotalSeconds() {
+ return totalSeconds;
+ }
- private ObjectMapper mapper;
+ public int getTotalOk() {
+ return totalOk;
+ }
++
++ private ObjectMapper mapper = new StreamsJacksonMapper();
- private ElasticsearchWriterConfiguration config;
+ public int getTotalFailed() {
+ return totalFailed;
+ }
- public ElasticsearchPersistWriter() {
- Config config = StreamsConfigurator.config.getConfig("elasticsearch");
- this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+ public int getTotalBatchCount() {
+ return totalBatchCount;
}
- public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
- this.config = config;
+ public long getTotalSizeInBytes() {
+ return totalSizeInBytes;
+ }
+
+ public long getBatchSizeInBytes() {
+ return batchSizeInBytes;
+ }
+
+ public int getBatchItemsSent() {
+ return batchItemsSent;
+ }
+
+ public List<String> getAffectedIndexes() {
+ return this.affectedIndexes;
+ }
+
+ public void setFlushThresholdSizeInBytes(long sizeInBytes) {
+ this.flushThresholdSizeInBytes = sizeInBytes;
}
+ public boolean isConnected() {
+ return (client != null);
+ }
+
++ private ElasticsearchWriterConfiguration config;
++
+ private static final int BYTES_IN_MB = 1024*1024;
+ private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+ private volatile int totalByteCount = 0;
+ private volatile int byteCount = 0;
-
- public boolean isConnected() { return (client != null); }
-
++
@Override
public void write(StreamsDatum streamsDatum) {
@@@ -280,10 -238,13 +293,11 @@@
// reset the current batch statistics
this.batchSizeInBytes = 0;
this.batchItemsSent = 0;
+ this.currentItems = 0;
- try
- {
+ try {
int count = 0;
- if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
- {
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
/****************************************************************************
* Author:
* Smashew
@@@ -381,8 -402,60 +395,60 @@@
}
}
+ private void trackItemAndBytesWritten(long sizeInBytes)
+ {
+ currentItems++;
+ batchItemsSent++;
+ batchSizeInBytes += sizeInBytes;
+
+ // If our queue is larger than our flush threashold, then we should flush the queue.
+ if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+ (currentItems >= batchSize) )
+ flushInternal();
+ }
+
+ private void checkAndCreateBulkRequest()
+ {
+ // Synchronize to ensure that we don't lose any records
+ synchronized (this)
+ {
+ if(bulkRequest == null)
+ bulkRequest = this.manager.getClient().prepareBulk();
+ }
+ }
+
+ private void checkIndexImplications(String indexName)
+ {
+
+ // check to see if we have seen this index before.
+ if(this.affectedIndexes.contains(indexName))
+ return;
+
+ // we haven't log this index.
+ this.affectedIndexes.add(indexName);
+
+ // Check to see if we are in 'veryLargeBulk' mode
+ // if we aren't, exit early
+ if(!this.veryLargeBulk)
+ return;
+
+
+ // They are in 'very large bulk' mode we want to turn off refreshing the index.
+
+ // Create a request then add the setting to tell it to stop refreshing the interval
+ UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
+ updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
+
+ // submit to ElasticSearch
+ this.manager.getClient()
+ .admin()
+ .indices()
+ .updateSettings(updateSettingsRequest)
+ .actionGet();
+ }
+
public void createIndexIfMissing(String indexName) {
- if(!this.manager.getClient()
+ if (!this.manager.getClient()
.admin()
.indices()
.exists(new IndicesExistsRequest(indexName))
@@@ -446,91 -524,20 +512,99 @@@
return toReturn;
}
+ @Override
+ public void prepare(Object configurationObject) {
+ mapper = StreamsJacksonMapper.getInstance();
+ veryLargeBulk = this.config.getBulk();
+ batchSize = this.config.getBatchSize();
+ start();
+ }
++
+ private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+ bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
+ @Override
+ public void onResponse(BulkResponse bulkItemResponses) {
+ if (bulkItemResponses.hasFailures())
+ LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- DatumStatusCounter counters = new DatumStatusCounter();
- counters.incrementAttempt(this.batchItemsSent);
- counters.incrementStatus(DatumStatus.SUCCESS, this.totalOk);
- counters.incrementStatus(DatumStatus.FAIL, this.totalFailed);
- return counters;
+ long thisFailed = 0;
+ long thisOk = 0;
+ long thisMillis = bulkItemResponses.getTookInMillis();
+
+ // keep track of the number of totalFailed and items that we have totalOk.
+ for (BulkItemResponse resp : bulkItemResponses.getItems()) {
+ if (resp.isFailed())
+ thisFailed++;
+ else
+ thisOk++;
+ }
+
+ totalAttempted += thisSent;
+ totalOk += thisOk;
+ totalFailed += thisFailed;
+ totalSeconds += (thisMillis / 1000);
+
+ if (thisSent != (thisOk + thisFailed))
+ LOGGER.error("We sent more items than this");
+
+ LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
+ MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
+ MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+ }
+
+ @Override
+ public void onFailure(Throwable e) {
+ LOGGER.error("Error bulk loading: {}", e.getMessage());
+ e.printStackTrace();
+ }
+ });
+
+ this.notify();
+ }
+
+ private void trackItemAndBytesWritten(long sizeInBytes) {
+ batchItemsSent++;
+ batchSizeInBytes += sizeInBytes;
+
+ // If our queue is larger than our flush threashold, then we should flush the queue.
+ if (batchSizeInBytes > flushThresholdSizeInBytes)
+ flushInternal();
+ }
+
+ private void checkAndCreateBulkRequest() {
+ // Synchronize to ensure that we don't lose any records
+ synchronized (this) {
+ if (bulkRequest == null)
+ bulkRequest = this.manager.getClient().prepareBulk();
+ }
+ }
+
+ private void checkIndexImplications(String indexName) {
+
+ // check to see if we have seen this index before.
+ if (this.affectedIndexes.contains(indexName))
+ return;
+
+ // we haven't log this index.
+ this.affectedIndexes.add(indexName);
+
+ // Check to see if we are in 'veryLargeBulk' mode
+ // if we aren't, exit early
+ if (!this.veryLargeBulk)
+ return;
+
+
+ // They are in 'very large bulk' mode we want to turn off refreshing the index.
+
+ // Create a request then add the setting to tell it to stop refreshing the interval
+ UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
+ updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
+
+ // submit to ElasticSearch
+ this.manager.getClient()
+ .admin()
+ .indices()
+ .updateSettings(updateSettingsRequest)
+ .actionGet();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index b456fa4,20ee951..db1ec76
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@@ -1,7 -1,8 +1,9 @@@
package org.apache.streams.twitter.provider;
+ import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
+import com.google.common.collect.Queues;
+ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.typesafe.config.Config;
@@@ -20,11 -21,10 +23,13 @@@ import twitter4j.json.DataObjectFactory
import java.io.Serializable;
import java.math.BigInteger;
- import java.util.*;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Queue;
-import java.util.concurrent.*;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
/**
* Created by sblackmon on 12/10/13.
@@@ -47,16 -47,22 +52,20 @@@ public class TwitterTimelineProvider im
this.config = config;
}
- protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>();
+ protected final Queue<StreamsDatum> providerQueue = Queues.synchronizedQueue(new ArrayBlockingQueue<StreamsDatum>(500));
+ protected int idsCount;
+ protected Twitter client;
protected Iterator<Long> ids;
- ListenableFuture providerTaskComplete;
-//
-// public BlockingQueue<Object> getInQueue() {
-// return inQueue;
-// }
-
protected ListeningExecutorService executor;
protected DateTime start;
protected DateTime end;
++ Boolean jsonStoreEnabled;
++ Boolean includeEntitiesEnabled;
++
private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
return new ThreadPoolExecutor(nThreads, nThreads,
5000L, TimeUnit.MILLISECONDS,
@@@ -88,12 -93,36 +96,13 @@@
return this.providerQueue;
}
-// public void run() {
-//
-// LOGGER.info("{} Running", STREAMS_ID);
-//
-// while( ids.hasNext() ) {
-// Long currentId = ids.next();
-// LOGGER.info("Provider Task Starting: {}", currentId);
-// captureTimeline(currentId);
-// }
-//
-// LOGGER.info("{} Finished. Cleaning up...", STREAMS_ID);
-//
-// client.shutdown();
-//
-// LOGGER.info("{} Exiting", STREAMS_ID);
-//
-// while(!providerTaskComplete.isDone() && !providerTaskComplete.isCancelled() ) {
-// try {
-// Thread.sleep(100);
-// } catch (InterruptedException e) {}
-// }
-// }
-
@Override
public void startStream() {
- // no op
+ LOGGER.debug("{} startStream", STREAMS_ID);
+ throw new org.apache.commons.lang.NotImplementedException();
}
- private void captureTimeline(long currentId) {
+ protected void captureTimeline(long currentId) {
Paging paging = new Paging(1, 200);
List<Status> statuses = null;
@@@ -223,14 -251,14 +235,19 @@@
Preconditions.checkNotNull(config.getOauth().getConsumerSecret());
Preconditions.checkNotNull(config.getOauth().getAccessToken());
Preconditions.checkNotNull(config.getOauth().getAccessTokenSecret());
-
Preconditions.checkNotNull(config.getFollow());
- Boolean jsonStoreEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getJsonStoreEnabled()))).or(true);
- Boolean includeEntitiesEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getIncludeEntities()))).or(true);
-
+ idsCount = config.getFollow().size();
ids = config.getFollow().iterator();
+
++ jsonStoreEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getJsonStoreEnabled()))).or(true);
++ includeEntitiesEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getIncludeEntities()))).or(true);
++
+ }
+
++
+ protected Twitter getTwitterClient()
+ {
String baseUrl = "https://api.twitter.com:443/1.1/";
ConfigurationBuilder builder = new ConfigurationBuilder()
@@@ -243,9 -271,11 +260,10 @@@
.setAsyncNumThreads(3)
.setRestBaseURL(baseUrl)
.setIncludeMyRetweetEnabled(Boolean.TRUE)
+ .setIncludeRTsEnabled(Boolean.TRUE)
.setPrettyDebugEnabled(Boolean.TRUE);
- client = new TwitterFactory(builder.build()).getInstance();
-
+ return new TwitterFactory(builder.build()).getInstance();
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
[04/14] git commit: Fixed deadlock issue
Posted by sb...@apache.org.
Fixed deadlock issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/fb8f7aea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/fb8f7aea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/fb8f7aea
Branch: refs/heads/master
Commit: fb8f7aea3032296a4c0dd779aca976104de44d61
Parents: 6934c7a
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 19:53:18 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 19:53:18 2014 -0400
----------------------------------------------------------------------
.../ElasticsearchPersistWriter.java | 20 +++++++++++++-------
1 file changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/fb8f7aea/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index c574eeb..5756e1c 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -54,7 +54,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private static final long WAITING_DOCS_LIMIT = 10000;
private static final int BYTES_IN_MB = 1024 * 1024;
private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
- private static final long MAX_MS_BEFORE_FLUSH = 10000;
+ private static final long DEFAULT_MAX_WAIT = 10000;
private final List<String> affectedIndexes = new ArrayList<String>();
private final ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
@@ -251,7 +251,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
@Override
public void prepare(Object configurationObject) {
- maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? MAX_MS_BEFORE_FLUSH : config.getMaxTimeBetweenFlushMs();
+ maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? DEFAULT_MAX_WAIT : config.getMaxTimeBetweenFlushMs();
mapper = StreamsJacksonMapper.getInstance();
start();
}
@@ -284,7 +284,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
public void flushInternal() {
- //This is unlocked in the callback
lock.writeLock().lock();
// we do not have a working bulk request, we can just exit here.
if (this.bulkRequest == null || batchItemsSent == 0)
@@ -339,6 +338,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
} catch (Exception e) {
LOGGER.info("We were broken from our loop: {}", e.getMessage());
+ } finally {
+ lock.writeLock().unlock();
}
}
@@ -472,11 +473,20 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
return toReturn;
}
+ /**
+ * This method is to ONLY be called by flushInternal otherwise the counts will be off.
+ * @param bulkRequest
+ * @param thisSent
+ * @param thisSizeInBytes
+ */
private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+ final Object messenger = new Object();
LOGGER.debug("Attempting to write {} items to ES", thisSent);
bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkItemResponses) {
+ lastWrite.set(System.currentTimeMillis());
+
if (bulkItemResponses.hasFailures())
LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
@@ -500,9 +510,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
if (thisSent != (thisOk + thisFailed))
LOGGER.error("We sent more items than this");
- lastWrite.set(System.currentTimeMillis());
- lock.writeLock().unlock();
-
LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
@@ -510,7 +517,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
@Override
public void onFailure(Throwable e) {
- lock.writeLock().unlock();
LOGGER.error("Error bulk loading: {}", e.getMessage());
e.printStackTrace();
}
[02/14] git commit: Updated Persist writer to have a configurable
background flush thread
Posted by sb...@apache.org.
Updated Persist writer to have a configurable background flush thread
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/c837a2cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/c837a2cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/c837a2cf
Branch: refs/heads/master
Commit: c837a2cf8800efd324003ba5a9299c16b9f8701a
Parents: e6ffe29
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 13:59:32 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 13:59:32 2014 -0400
----------------------------------------------------------------------
.../ElasticsearchConfigurator.java | 4 +-
.../ElasticsearchPersistWriter.java | 184 +++++++++++--------
.../ElasticsearchWriterConfiguration.json | 6 +-
3 files changed, 116 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c837a2cf/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 483a177..8a343a6 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory;
import java.util.List;
/**
- * Created by sblackmon on 12/10/13.
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
*/
public class ElasticsearchConfigurator {
@@ -51,9 +51,11 @@ public class ElasticsearchConfigurator {
String index = elasticsearch.getString("index");
String type = elasticsearch.getString("type");
+ Long maxMsBeforeFlush = elasticsearch.hasPath("MaxTimeBetweenFlushMs") ? elasticsearch.getLong("MaxTimeBetweenFlushMs") : null;
elasticsearchWriterConfiguration.setIndex(index);
elasticsearchWriterConfiguration.setType(type);
+ elasticsearchWriterConfiguration.setMaxTimeBetweenFlushMs(maxMsBeforeFlush);
return elasticsearchWriterConfiguration;
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c837a2cf/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ff7e0f5..c574eeb 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -36,10 +36,15 @@ import java.io.OutputStreamWriter;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable {
public static final String STREAMS_ID = "ElasticsearchPersistWriter";
-
public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
@@ -49,8 +54,11 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private static final long WAITING_DOCS_LIMIT = 10000;
private static final int BYTES_IN_MB = 1024 * 1024;
private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+ private static final long MAX_MS_BEFORE_FLUSH = 10000;
private final List<String> affectedIndexes = new ArrayList<String>();
+ private final ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
private ObjectMapper mapper = new StreamsJacksonMapper();
private ElasticsearchClientManager manager;
@@ -61,6 +69,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private OutputStreamWriter currentWriter = null;
private int batchSize = 50;
private int totalRecordsWritten = 0;
+ private long maxMsBeforeFlush;
private boolean veryLargeBulk = false; // by default this setting is set to false
protected Thread task;
@@ -78,10 +87,11 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private volatile int batchItemsSent = 0;
private volatile int totalByteCount = 0;
private volatile int byteCount = 0;
+ private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
public ElasticsearchPersistWriter() {
Config config = StreamsConfigurator.config.getConfig("elasticsearch");
- this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+ this.config = ElasticsearchConfigurator.detectWriterConfiguration(config);
}
public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
@@ -173,6 +183,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
try {
flush();
+ backgroundFlushTask.shutdownNow();
} catch (IOException e) {
e.printStackTrace();
}
@@ -240,6 +251,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
@Override
public void prepare(Object configurationObject) {
+ maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? MAX_MS_BEFORE_FLUSH : config.getMaxTimeBetweenFlushMs();
mapper = StreamsJacksonMapper.getInstance();
start();
}
@@ -254,7 +266,17 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
public void start() {
-
+ backgroundFlushTask.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ LOGGER.debug("Checking to see if data needs to be flushed");
+ long time = System.currentTimeMillis() - lastWrite.get();
+ if (time > maxMsBeforeFlush && batchItemsSent > 0) {
+ LOGGER.debug("Background Flush task determined {} are waiting to be flushed. It has been {} since the last write to ES", batchItemsSent, time);
+ flushInternal();
+ }
+ }
+ }, 0, maxMsBeforeFlush * 2, TimeUnit.MILLISECONDS);
manager = new ElasticsearchClientManager(config);
client = manager.getClient();
@@ -262,62 +284,63 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
public void flushInternal() {
- synchronized (this) {
- // we do not have a working bulk request, we can just exit here.
- if (this.bulkRequest == null || batchItemsSent == 0)
- return;
+ //This is unlocked in the callback
+ lock.writeLock().lock();
+ // we do not have a working bulk request, we can just exit here.
+ if (this.bulkRequest == null || batchItemsSent == 0)
+ return;
- // call the flush command.
- flush(this.bulkRequest, batchItemsSent, batchSizeInBytes);
+ // call the flush command.
+ flush(this.bulkRequest, batchItemsSent, batchSizeInBytes);
- // null the flush request, this will be created in the 'add' function below
- this.bulkRequest = null;
+ // null the flush request, this will be created in the 'add' function below
+ this.bulkRequest = null;
- // record the proper statistics, and add it to our totals.
- this.totalSizeInBytes += this.batchSizeInBytes;
- this.totalSent += batchItemsSent;
+ // record the proper statistics, and add it to our totals.
+ this.totalSizeInBytes += this.batchSizeInBytes;
+ this.totalSent += batchItemsSent;
- // reset the current batch statistics
- this.batchSizeInBytes = 0;
- this.batchItemsSent = 0;
+ // reset the current batch statistics
+ this.batchSizeInBytes = 0;
+ this.batchItemsSent = 0;
- try {
- int count = 0;
- if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
- /****************************************************************************
- * Author:
- * Smashew
- *
- * Date:
- * 2013-10-20
- *
- * Note:
- * With the information that we have on hand. We need to develop a heuristic
- * that will determine when the cluster is having a problem indexing records
- * by telling it to pause and wait for it to catch back up. A
- *
- * There is an impact to us, the caller, whenever this happens as well. Items
- * that are not yet fully indexed by the server sit in a queue, on the client
- * that can cause the heap to overflow. This has been seen when re-indexing
- * large amounts of data to a small cluster. The "deletes" + "indexes" can
- * cause the server to have many 'outstandingItems" in queue. Running this
- * software with large amounts of data, on a small cluster, can re-create
- * this problem.
- *
- * DO NOT DELETE THESE LINES
- ****************************************************************************/
-
- // wait for the flush to catch up. We are going to cap this at
- while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
- Thread.sleep(10);
-
- if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
- LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
- }
- } catch (Exception e) {
- LOGGER.info("We were broken from our loop: {}", e.getMessage());
+ try {
+ int count = 0;
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
+ /****************************************************************************
+ * Author:
+ * Smashew
+ *
+ * Date:
+ * 2013-10-20
+ *
+ * Note:
+ * With the information that we have on hand. We need to develop a heuristic
+ * that will determine when the cluster is having a problem indexing records
+ * by telling it to pause and wait for it to catch back up. A
+ *
+ * There is an impact to us, the caller, whenever this happens as well. Items
+ * that are not yet fully indexed by the server sit in a queue, on the client
+ * that can cause the heap to overflow. This has been seen when re-indexing
+ * large amounts of data to a small cluster. The "deletes" + "indexes" can
+ * cause the server to have many 'outstandingItems" in queue. Running this
+ * software with large amounts of data, on a small cluster, can re-create
+ * this problem.
+ *
+ * DO NOT DELETE THESE LINES
+ ****************************************************************************/
+
+ // wait for the flush to catch up. We are going to cap this at
+ while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
+ Thread.sleep(10);
+
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
+ LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
}
+ } catch (Exception e) {
+ LOGGER.info("We were broken from our loop: {}", e.getMessage());
}
+
}
public void add(String indexName, String type, String json) {
@@ -353,32 +376,35 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
public void add(UpdateRequest updateRequest) {
Preconditions.checkNotNull(updateRequest);
- synchronized (this) {
- checkAndCreateBulkRequest();
- checkIndexImplications(updateRequest.index());
- bulkRequest.add(updateRequest);
- try {
- Optional<Integer> size = Objects.firstNonNull(
- Optional.fromNullable(updateRequest.doc().source().length()),
- Optional.fromNullable(updateRequest.script().length()));
- trackItemAndBytesWritten(size.get().longValue());
- } catch (NullPointerException x) {
- trackItemAndBytesWritten(1000);
- }
+ lock.writeLock().lock();
+ checkAndCreateBulkRequest();
+ checkIndexImplications(updateRequest.index());
+ bulkRequest.add(updateRequest);
+ try {
+ Optional<Integer> size = Objects.firstNonNull(
+ Optional.fromNullable(updateRequest.doc().source().length()),
+ Optional.fromNullable(updateRequest.script().length()));
+ trackItemAndBytesWritten(size.get().longValue());
+ } catch (NullPointerException x) {
+ trackItemAndBytesWritten(1000);
+ } finally {
+ lock.writeLock().unlock();
}
}
public void add(IndexRequest indexRequest) {
- synchronized (this) {
- checkAndCreateBulkRequest();
- checkIndexImplications(indexRequest.index());
- bulkRequest.add(indexRequest);
- try {
- trackItemAndBytesWritten(indexRequest.source().length());
- } catch (NullPointerException x) {
- LOGGER.warn("NPE adding/sizing indexrequest");
- }
+ lock.writeLock().lock();
+ checkAndCreateBulkRequest();
+ checkIndexImplications(indexRequest.index());
+ bulkRequest.add(indexRequest);
+ try {
+ trackItemAndBytesWritten(indexRequest.source().length());
+ } catch (NullPointerException x) {
+ LOGGER.warn("NPE adding/sizing indexrequest");
+ } finally {
+ lock.writeLock().unlock();
}
+
}
public void createIndexIfMissing(String indexName) {
@@ -447,6 +473,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+ LOGGER.debug("Attempting to write {} items to ES", thisSent);
bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkItemResponses) {
@@ -473,6 +500,9 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
if (thisSent != (thisOk + thisFailed))
LOGGER.error("We sent more items than this");
+ lastWrite.set(System.currentTimeMillis());
+ lock.writeLock().unlock();
+
LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
@@ -480,12 +510,11 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
@Override
public void onFailure(Throwable e) {
+ lock.writeLock().unlock();
LOGGER.error("Error bulk loading: {}", e.getMessage());
e.printStackTrace();
}
});
-
- this.notify();
}
private void trackItemAndBytesWritten(long sizeInBytes) {
@@ -499,9 +528,12 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private void checkAndCreateBulkRequest() {
// Synchronize to ensure that we don't lose any records
- synchronized (this) {
+ lock.writeLock().lock();
+ try {
if (bulkRequest == null)
bulkRequest = this.manager.getClient().prepareBulk();
+ } finally {
+ lock.writeLock().unlock();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c837a2cf/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 21aad5c..c56aa82 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@ -13,6 +13,10 @@
"type": {
"type": "string",
"description": "Type to write as"
- }
+ },
+ "MaxTimeBetweenFlushMs": {
+ "type": "String",
+ "format": "utc-millisec"
+ }
}
}
\ No newline at end of file
[12/14] git commit: increasing queue size and initializing twitter
during prepare
Posted by sb...@apache.org.
increasing queue size and initializing twitter during prepare
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/64cd1227
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/64cd1227
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/64cd1227
Branch: refs/heads/master
Commit: 64cd1227caba30021f1cfc0d19e5521ce4c80cf4
Parents: 87bd8a3
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 19:27:39 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 19:27:39 2014 -0500
----------------------------------------------------------------------
.../apache/streams/twitter/provider/TwitterTimelineProvider.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/64cd1227/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index 3799737..e4b0460 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -54,7 +54,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
this.config = config;
}
- protected final Queue<StreamsDatum> providerQueue = Queues.synchronizedQueue(new ArrayBlockingQueue<StreamsDatum>(500));
+ protected final Queue<StreamsDatum> providerQueue = Queues.synchronizedQueue(new ArrayBlockingQueue<StreamsDatum>(5000));
protected int idsCount;
protected Twitter client;
@@ -245,6 +245,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
jsonStoreEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getJsonStoreEnabled()))).or(true);
includeEntitiesEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getIncludeEntities()))).or(true);
+ client = getTwitterClient();
}
[09/14] git commit: was missing imports and using random in a way
that wouldn't build in jdk6
Posted by sb...@apache.org.
was missing imports and using random in a way that wouldn't build in jdk6
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1cc9d0bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1cc9d0bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1cc9d0bb
Branch: refs/heads/master
Commit: 1cc9d0bb8b54f24f20fa979ba84e9269603b896e
Parents: 145ec84
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 14:32:14 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 14:32:14 2014 -0500
----------------------------------------------------------------------
.../streams/twitter/provider/TwitterTimelineProvider.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1cc9d0bb/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index db1ec76..799abe7 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -23,8 +23,10 @@ import twitter4j.json.DataObjectFactory;
import java.io.Serializable;
import java.math.BigInteger;
+import java.lang.Math;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Queue;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ExecutorService;
@@ -154,7 +156,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
try {
// wait one tenth of a millisecond
Thread.yield();
- Thread.sleep(new Random().nextInt(2));
+ Thread.sleep(1);
Thread.yield();
}
catch(IllegalArgumentException e) {
@@ -260,7 +262,6 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
.setAsyncNumThreads(3)
.setRestBaseURL(baseUrl)
.setIncludeMyRetweetEnabled(Boolean.TRUE)
- .setIncludeRTsEnabled(Boolean.TRUE)
.setPrettyDebugEnabled(Boolean.TRUE);
return new TwitterFactory(builder.build()).getInstance();
[13/14] git commit: line missing from merge
Posted by sb...@apache.org.
line missing from merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/237f2a6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/237f2a6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/237f2a6a
Branch: refs/heads/master
Commit: 237f2a6ab13c68a61cf715004fe7b85db66237f6
Parents: 64cd122
Author: sblackmon <sb...@w2odigital.com>
Authored: Fri May 9 15:44:42 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Fri May 9 15:44:42 2014 -0500
----------------------------------------------------------------------
.../apache/streams/elasticsearch/ElasticsearchPersistWriter.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/237f2a6a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 97796e2..271521c 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -418,8 +418,10 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
// If our queue is larger than our flush threashold, then we should flush the queue.
if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
- (currentItems >= batchSize) )
+ (currentItems >= batchSize) ) {
flushInternal();
+ this.currentItems = 0;
+ }
}
private void checkIndexImplications(String indexName)