You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2014/05/13 17:22:51 UTC

[01/14] git commit: userstream now writing to ES in real-time

Repository: incubator-streams
Updated Branches:
  refs/heads/master c38eef6cf -> e68300936


userstream now writing to ES in real-time


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/522096e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/522096e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/522096e8

Branch: refs/heads/master
Commit: 522096e8a914ea738a97a536ef19a1416453e5f0
Parents: f877c5f
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Mon Apr 14 10:25:27 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Mon Apr 14 10:25:27 2014 -0500

----------------------------------------------------------------------
 .../streams-persist-elasticsearch/pom.xml       |  1 -
 .../ElasticsearchConfigurator.java              |  7 ++
 .../ElasticsearchPersistWriter.java             | 25 ++++---
 .../ElasticsearchWriterConfiguration.json       | 10 +++
 .../twitter/processor/TwitterTypeConverter.java |  2 +-
 .../twitter/provider/TwitterStreamProvider.java | 69 +++++++++-----------
 .../provider/TwitterTimelineProvider.java       | 32 ++++-----
 .../local/tasks/StreamsProviderTask.java        |  2 +-
 8 files changed, 78 insertions(+), 70 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/pom.xml b/streams-contrib/streams-persist-elasticsearch/pom.xml
index 07433ba..95f3357 100644
--- a/streams-contrib/streams-persist-elasticsearch/pom.xml
+++ b/streams-contrib/streams-persist-elasticsearch/pom.xml
@@ -62,7 +62,6 @@
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>build-helper-maven-plugin</artifactId>
-                <version>1.8</version>
                 <executions>
                     <execution>
                         <id>add-source</id>

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 20b5c08..e6bf3ca 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@ -52,9 +52,16 @@ public class ElasticsearchConfigurator {
         String index = elasticsearch.getString("index");
         String type = elasticsearch.getString("type");
 
+        if( elasticsearch.hasPath("bulk"))
+            elasticsearchWriterConfiguration.setBulk(elasticsearch.getBoolean("bulk"));
+
+        if( elasticsearch.hasPath("batchSize"))
+            elasticsearchWriterConfiguration.setBatchSize(elasticsearch.getLong("batchSize"));
+
         elasticsearchWriterConfiguration.setIndex(index);
         elasticsearchWriterConfiguration.setType(type);
 
+
         return elasticsearchWriterConfiguration;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ad89013..405d1b8 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -51,34 +51,27 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private BulkRequestBuilder bulkRequest;
     private OutputStreamWriter currentWriter = null;
 
-    private int batchSize = 50;
-    private int totalRecordsWritten = 0;
-    private boolean veryLargeBulk = false;  // by default this setting is set to false
+    private long batchSize;
+    private boolean veryLargeBulk;  // by default this setting is set to false
 
     private final static Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
     private static final long WAITING_DOCS_LIMIT = 10000;
 
     public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
 
+    private volatile int currentItems = 0;
     private volatile int totalSent = 0;
     private volatile int totalSeconds = 0;
     private volatile int totalAttempted = 0;
     private volatile int totalOk = 0;
     private volatile int totalFailed = 0;
     private volatile int totalBatchCount = 0;
+    private volatile int totalRecordsWritten = 0;
     private volatile long totalSizeInBytes = 0;
 
     private volatile long batchSizeInBytes = 0;
     private volatile int batchItemsSent = 0;
 
-    public void setBatchSize(int batchSize) {
-        this.batchSize = batchSize;
-    }
-
-    public void setVeryLargeBulk(boolean veryLargeBulk) {
-        this.veryLargeBulk = veryLargeBulk;
-    }
-
     private final List<String> affectedIndexes = new ArrayList<String>();
 
     public int getTotalOutstanding()                           { return this.totalSent - (this.totalFailed + this.totalOk); }
@@ -99,7 +92,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     protected volatile Queue<StreamsDatum> persistQueue;
 
-    private ObjectMapper mapper = new StreamsJacksonMapper();
+    private ObjectMapper mapper;
 
     private ElasticsearchWriterConfiguration config;
 
@@ -245,6 +238,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
             // reset the current batch statistics
             this.batchSizeInBytes = 0;
             this.batchItemsSent = 0;
+            this.currentItems = 0;
 
             try
             {
@@ -323,6 +317,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
                 LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
                         MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
                         MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+
             }
 
             @Override
@@ -409,11 +404,13 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     private void trackItemAndBytesWritten(long sizeInBytes)
     {
+        currentItems++;
         batchItemsSent++;
         batchSizeInBytes += sizeInBytes;
 
         // If our queue is larger than our flush threashold, then we should flush the queue.
-        if(batchSizeInBytes > flushThresholdSizeInBytes)
+        if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+                (currentItems >= batchSize) )
             flushInternal();
     }
 
@@ -530,6 +527,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     @Override
     public void prepare(Object configurationObject) {
         mapper = StreamsJacksonMapper.getInstance();
+        veryLargeBulk = this.config.getBulk();
+        batchSize = this.config.getBatchSize();
         start();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 21aad5c..5e169b7 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@ -13,6 +13,16 @@
         "type": {
             "type": "string",
             "description": "Type to write as"
+        },
+        "bulk": {
+            "type": "boolean",
+            "description": "Index in large or small batches",
+            "default": "false"
+        },
+        "batchSize": {
+            "type": "integer",
+            "description": "Item Count before flush",
+            "default": 100
         }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
index 68820c9..5e88bdd 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/processor/TwitterTypeConverter.java
@@ -27,7 +27,7 @@ import java.util.Queue;
  */
 public class TwitterTypeConverter implements StreamsProcessor {
 
-    private final static String STREAMS_ID = "TwitterTypeConverter";
+    public final static String STREAMS_ID = "TwitterTypeConverter";
 
     private final static Logger LOGGER = LoggerFactory.getLogger(TwitterTypeConverter.class);
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
index 3df7d02..ba88803 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
@@ -1,10 +1,7 @@
 package org.apache.streams.twitter.provider;
 
-import com.google.common.base.Joiner;
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
 import com.google.common.collect.Queues;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -19,10 +16,9 @@ import com.twitter.hbc.httpclient.auth.Authentication;
 import com.twitter.hbc.httpclient.auth.BasicAuth;
 import com.twitter.hbc.httpclient.auth.OAuth1;
 import com.typesafe.config.Config;
+import org.apache.commons.lang.NotImplementedException;
 import org.apache.streams.config.StreamsConfigurator;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.core.StreamsProvider;
-import org.apache.streams.core.StreamsResultSet;
+import org.apache.streams.core.*;
 import org.apache.streams.twitter.TwitterStreamConfiguration;
 import org.apache.streams.twitter.processor.TwitterEventProcessor;
 import org.joda.time.DateTime;
@@ -31,7 +27,6 @@ import org.slf4j.LoggerFactory;
 
 import java.io.Serializable;
 import java.math.BigInteger;
-import java.util.Collection;
 import java.util.List;
 import java.util.Queue;
 import java.util.concurrent.*;
@@ -39,7 +34,7 @@ import java.util.concurrent.*;
 /**
  * Created by sblackmon on 12/10/13.
  */
-public class TwitterStreamProvider implements StreamsProvider, Serializable {
+public class TwitterStreamProvider implements StreamsProvider, Serializable, DatumStatusCountable {
 
     public final static String STREAMS_ID = "TwitterStreamProvider";
 
@@ -47,8 +42,6 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
 
     private TwitterStreamConfiguration config;
 
-    private Class klass;
-
     public TwitterStreamConfiguration getConfig() {
         return config;
     }
@@ -57,9 +50,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
         this.config = config;
     }
 
-    protected BlockingQueue inQueue = new LinkedBlockingQueue<String>(10000);
+    protected BlockingQueue hosebirdQueue = new LinkedBlockingQueue<String>(1000);
 
-    protected volatile Queue<StreamsDatum> providerQueue;
+    protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>(1000);
 
     protected Hosts hosebirdHosts;
     protected Authentication auth;
@@ -68,6 +61,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
 
     protected ListeningExecutorService executor;
 
+    private DatumStatusCounter countersCurrent = new DatumStatusCounter();
+    private DatumStatusCounter countersTotal = new DatumStatusCounter();
+
     private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
         return new ThreadPoolExecutor(nThreads, nThreads,
                 5000L, TimeUnit.MILLISECONDS,
@@ -83,50 +79,43 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
         this.config = config;
     }
 
-    public TwitterStreamProvider(Class klass) {
-        Config config = StreamsConfigurator.config.getConfig("twitter");
-        this.config = TwitterStreamConfigurator.detectConfiguration(config);
-        this.klass = klass;
-        providerQueue = new LinkedBlockingQueue<StreamsDatum>();
-    }
-
-    public TwitterStreamProvider(TwitterStreamConfiguration config, Class klass) {
-        this.config = config;
-        this.klass = klass;
-        providerQueue = new LinkedBlockingQueue<StreamsDatum>();
-
-    }
-
     @Override
     public void startStream() {
 
         for (int i = 0; i < 5; i++) {
-            executor.submit(new TwitterEventProcessor(inQueue, providerQueue, klass));
+            executor.submit(new TwitterEventProcessor(hosebirdQueue, providerQueue, String.class));
         }
 
         new Thread(new TwitterStreamProviderTask(this)).start();
+
     }
 
     @Override
     public synchronized StreamsResultSet readCurrent() {
-        Collection<StreamsDatum> currentIterator = Lists.newArrayList();
-        Iterators.addAll(currentIterator, providerQueue.iterator());
 
-        StreamsResultSet current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(currentIterator));
+        StreamsResultSet current;
 
-        providerQueue.clear();
+        synchronized( TwitterStreamProvider.class ) {
+            current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(providerQueue));
+            current.setCounter(new DatumStatusCounter());
+            current.getCounter().add(countersCurrent);
+            countersTotal.add(countersCurrent);
+            countersCurrent = new DatumStatusCounter();
+            providerQueue.clear();
+        }
 
         return current;
     }
 
     @Override
     public StreamsResultSet readNew(BigInteger sequence) {
-        return null;
+        throw new NotImplementedException();
     }
 
     @Override
-    public StreamsResultSet readRange(DateTime start, DateTime end) {
-        return null;
+    public StreamsResultSet readRange(DateTime start, DateTime end)
+    {
+        throw new NotImplementedException();
     }
 
     @Override
@@ -134,8 +123,6 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
 
         executor = MoreExecutors.listeningDecorator(newFixedThreadPoolWithQueueSize(5, 20));
 
-        Preconditions.checkNotNull(this.klass);
-
         Preconditions.checkNotNull(config.getEndpoint());
 
         if(config.getEndpoint().equals("userstream") ) {
@@ -212,7 +199,8 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
             .hosts(hosebirdHosts)
             .endpoint(endpoint)
             .authentication(auth)
-            .processor(new StringDelimitedProcessor(inQueue))
+            .connectionTimeout(1200000)
+            .processor(new StringDelimitedProcessor(hosebirdQueue))
             .build();
 
     }
@@ -220,7 +208,12 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable {
     @Override
     public void cleanUp() {
         for (int i = 0; i < 5; i++) {
-            inQueue.add(TwitterEventProcessor.TERMINATE);
+            hosebirdQueue.add(TwitterEventProcessor.TERMINATE);
         }
     }
+
+    @Override
+    public DatumStatusCounter getDatumStatusCounter() {
+        return countersTotal;
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index b9551ad..20ee951 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -2,10 +2,6 @@ package org.apache.streams.twitter.provider;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Iterators;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
@@ -25,7 +21,9 @@ import twitter4j.json.DataObjectFactory;
 
 import java.io.Serializable;
 import java.math.BigInteger;
-import java.util.*;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
 import java.util.concurrent.*;
 
 /**
@@ -120,7 +118,8 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
 
     @Override
     public void startStream() {
-        // no op
+        LOGGER.debug("{} startStream", STREAMS_ID);
+        throw new org.apache.commons.lang.NotImplementedException();
     }
 
     private void captureTimeline(long currentId) {
@@ -182,15 +181,20 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
     }
 
     public StreamsResultSet readCurrent() {
+        LOGGER.debug("{} readCurrent", STREAMS_ID);
 
         Preconditions.checkArgument(ids.hasNext());
 
-        LOGGER.info("readCurrent");
+        StreamsResultSet current;
+
+        synchronized( TwitterTimelineProvider.class ) {
+
+            while( ids.hasNext() ) {
+                Long currentId = ids.next();
+                LOGGER.info("Provider Task Starting: {}", currentId);
+                captureTimeline(currentId);
+            }
 
-        while( ids.hasNext() ) {
-            Long currentId = ids.next();
-            LOGGER.info("Provider Task Starting: {}", currentId);
-            captureTimeline(currentId);
         }
 
         LOGGER.info("Finished.  Cleaning up...");
@@ -212,11 +216,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
 
     public StreamsResultSet readRange(DateTime start, DateTime end) {
         LOGGER.debug("{} readRange", STREAMS_ID);
-        this.start = start;
-        this.end = end;
-        readCurrent();
-        StreamsResultSet result = (StreamsResultSet)providerQueue.iterator();
-        return result;
+        throw new NotImplementedException();
     }
 
     void shutdownAndAwaitTermination(ExecutorService pool) {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/522096e8/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index 5cf515c..101b47c 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -32,7 +32,7 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
     private static final int START = 0;
     private static final int END = 1;
 
-    private static final int DEFAULT_TIMEOUT_MS = 30000;
+    private static final int DEFAULT_TIMEOUT_MS = 1000000;
 
     private StreamsProvider provider;
     private AtomicBoolean keepRunning;


[05/14] git commit: Fixed issue where exception terminated thread and it wouldn't start again

Posted by sb...@apache.org.
Fixed issue where exception terminated thread and it wouldn't start again


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/f62afa57
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/f62afa57
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/f62afa57

Branch: refs/heads/master
Commit: f62afa57ce5ca19d639c84e89ea29f998a16fbe5
Parents: fb8f7ae
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 19:54:56 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 19:54:56 2014 -0400

----------------------------------------------------------------------
 .../provider/SysomosHeartbeatStream.java        | 45 +++++++++++---------
 1 file changed, 26 insertions(+), 19 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f62afa57/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
index c5145fb..9cd5898 100644
--- a/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
+++ b/streams-contrib/streams-provider-sysomos/src/main/java/org/apache/streams/sysomos/provider/SysomosHeartbeatStream.java
@@ -72,29 +72,36 @@ public class SysomosHeartbeatStream implements Runnable {
     }
 
     protected QueryResult executeAPIRequest() {
-        BeatApi.BeatResponse response = this.client.createRequestBuilder()
-                .setHeartBeatId(heartbeatId)
-                .setOffset(0)
-                .setReturnSetSize(maxApiBatch).execute();
-
-        LOGGER.debug("Received {} results from API query", response.getCount());
+        BeatApi.BeatResponse response = null;
+        try {
+            response = this.client.createRequestBuilder()
+                    .setHeartBeatId(heartbeatId)
+                    .setOffset(0)
+                    .setReturnSetSize(maxApiBatch).execute();
+
+            LOGGER.debug("Received {} results from API query", response.getCount());
+        } catch (Exception e) {
+            LOGGER.warn("Error querying Sysomos API", e);
+        }
 
         String currentId = null;
         boolean matched = false;
-        for(BeatApi.BeatResponse.Beat beat : response.getBeat()) {
-            String docId = beat.getDocid();
-            //We get documents in descending time order.  This will set the id to the latest document
-            if(currentId == null) {
-                currentId = docId;
-            }
-            //We only want to process documents that we know we have not seen before
-            if(lastID != null && lastID.equals(docId)) {
-                matched = true;
-                break;
+        if(response != null) {
+            for (BeatApi.BeatResponse.Beat beat : response.getBeat()) {
+                String docId = beat.getDocid();
+                //We get documents in descending time order.  This will set the id to the latest document
+                if (currentId == null) {
+                    currentId = docId;
+                }
+                //We only want to process documents that we know we have not seen before
+                if (lastID != null && lastID.equals(docId)) {
+                    matched = true;
+                    break;
+                }
+                StreamsDatum item = new StreamsDatum(beat, docId);
+                item.getMetadata().put("heartbeat", this.heartbeatId);
+                this.provider.enqueueItem(item);
             }
-            StreamsDatum item = new StreamsDatum(beat, docId);
-            item.getMetadata().put("heartbeat", this.heartbeatId);
-            this.provider.enqueueItem(item);
         }
         return new QueryResult(matched, currentId);
     }


[10/14] git commit: Merge branch 'streamstutorial' of https://git-wip-us.apache.org/repos/asf/incubator-streams into streamstutorial

Posted by sb...@apache.org.
Merge branch 'streamstutorial' of https://git-wip-us.apache.org/repos/asf/incubator-streams into streamstutorial

Conflicts:
	streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
	streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/c59ab10e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/c59ab10e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/c59ab10e

Branch: refs/heads/master
Commit: c59ab10ed92c13d3b116c75f63a2fb921f0a43dc
Parents: 7e4e109 1cc9d0b
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 15:11:44 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 15:11:44 2014 -0500

----------------------------------------------------------------------
 .../ElasticsearchConfigurator.java              |   4 +-
 .../ElasticsearchPersistWriter.java             | 261 +++++++++++--------
 .../ElasticsearchWriterConfiguration.json       |   5 +-
 .../provider/SysomosHeartbeatStream.java        |  45 ++--
 .../provider/TwitterTimelineProvider.java       |  10 +-
 .../local/builders/LocalStreamBuilder.java      | 161 ++++++++----
 .../local/tasks/StreamsPersistWriterTask.java   |   6 +-
 .../local/tasks/StreamsProcessorTask.java       |   2 +-
 8 files changed, 304 insertions(+), 190 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c59ab10e/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index e99c4ff,ce23197..97796e2
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@@ -373,26 -396,30 +397,29 @@@ public class ElasticsearchPersistWrite
      }
  
      public void add(IndexRequest indexRequest) {
-         synchronized (this) {
-             checkAndCreateBulkRequest();
-             checkIndexImplications(indexRequest.index());
-             bulkRequest.add(indexRequest);
-             try {
-                 trackItemAndBytesWritten(indexRequest.source().length());
-             } catch (NullPointerException x) {
-                 LOGGER.warn("NPE adding/sizing indexrequest");
-             }
+         lock.writeLock().lock();
+         checkAndCreateBulkRequest();
+         checkIndexImplications(indexRequest.index());
+         bulkRequest.add(indexRequest);
+         try {
+             trackItemAndBytesWritten(indexRequest.source().length());
+         } catch (NullPointerException x) {
+             LOGGER.warn("NPE adding/sizing indexrequest");
+         } finally {
+             lock.writeLock().unlock();
          }
 -
      }
  
-     private void checkAndCreateBulkRequest()
+     private void trackItemAndBytesWritten(long sizeInBytes)
      {
-         // Synchronize to ensure that we don't lose any records
-         synchronized (this)
-         {
-             if(bulkRequest == null)
-                 bulkRequest = this.manager.getClient().prepareBulk();
-         }
+         currentItems++;
+         batchItemsSent++;
+         batchSizeInBytes += sizeInBytes;
+ 
+         // If our queue is larger than our flush threashold, then we should flush the queue.
+         if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+                 (currentItems >= batchSize) )
+             flushInternal();
      }
  
      private void checkIndexImplications(String indexName)

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c59ab10e/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index 06ba7be,799abe7..3799737
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@@ -22,8 -22,12 +22,12 @@@ import twitter4j.conf.ConfigurationBuil
  import twitter4j.json.DataObjectFactory;
  
  import java.io.Serializable;
 -import java.math.BigInteger;
+ import java.lang.Math;
 +import java.math.BigInteger;
- import java.util.*;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Queue;
  import java.util.concurrent.ArrayBlockingQueue;
  import java.util.concurrent.ExecutorService;
  import java.util.concurrent.ThreadPoolExecutor;


[11/14] git commit: fixed premature-timeout

Posted by sb...@apache.org.
fixed premature-timeout


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/87bd8a3f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/87bd8a3f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/87bd8a3f

Branch: refs/heads/master
Commit: 87bd8a3f874e711933b4e9c0a5e20518c9ff6f79
Parents: c59ab10
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 17:19:45 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 17:19:45 2014 -0500

----------------------------------------------------------------------
 .../java/org/apache/streams/local/tasks/StreamsProviderTask.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/87bd8a3f/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
index cd43d44..cc4844c 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
@@ -130,7 +130,8 @@ public class StreamsProviderTask extends BaseStreamsTask implements DatumStatusC
                                 zeros = 0;
                             }
                             flushResults(resultSet);
-                            if( zeros > (timeout / DEFAULT_SLEEP_TIME_MS))
+                            // the way this works needs to change...
+                            if( zeros > (timeout))
                                 this.keepRunning.set(false);
                             Thread.sleep(DEFAULT_SLEEP_TIME_MS);
                         } catch (InterruptedException e) {


[14/14] git commit: Merge commit '237f2a6ab13c68a61cf715004fe7b85db66237f6'

Posted by sb...@apache.org.
Merge commit '237f2a6ab13c68a61cf715004fe7b85db66237f6'


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/e6830093
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/e6830093
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/e6830093

Branch: refs/heads/master
Commit: e68300936e100409ba4b518499eee66634d96245
Parents: c38eef6 237f2a6
Author: Steve Blackmon <sb...@w2odigital.com>
Authored: Tue May 13 10:21:52 2014 -0500
Committer: Steve Blackmon <sb...@w2odigital.com>
Committed: Tue May 13 10:21:52 2014 -0500

----------------------------------------------------------------------
 .../streams-persist-elasticsearch/pom.xml       |   1 -
 .../ElasticsearchConfigurator.java              |  11 +-
 .../ElasticsearchPersistWriter.java             | 297 +++++++++++--------
 .../ElasticsearchWriterConfiguration.json       |  15 +-
 .../provider/SysomosHeartbeatStream.java        |  45 +--
 .../streams-provider-twitter/pom.xml            |  13 +
 .../twitter/processor/TwitterTypeConverter.java |   2 +-
 .../provider/TwitterEventClassifier.java        |  10 +-
 .../twitter/provider/TwitterStreamProvider.java |  69 +++--
 .../provider/TwitterTimelineProvider.java       |  56 ++--
 .../TwitterJsonActivitySerializer.java          |   3 +-
 .../TwitterJsonDeleteActivitySerializer.java    |   3 +-
 .../TwitterJsonRetweetActivitySerializer.java   |   3 +-
 .../TwitterJsonTweetActivitySerializer.java     |   7 +-
 ...erJsonUserstreameventActivitySerializer.java |  92 ++++++
 .../main/jsonschema/com/twitter/FriendList.json |  20 ++
 .../jsonschema/com/twitter/UserstreamEvent.json |  47 +++
 .../jackson/StreamsDateTimeDeserializer.java    |   3 +-
 .../jackson/StreamsDateTimeSerializer.java      |   5 +-
 .../jackson/StreamsPeriodDeserializer.java      |   3 +-
 .../jackson/StreamsPeriodSerializer.java        |   3 +-
 .../local/builders/LocalStreamBuilder.java      | 161 ++++++----
 .../local/tasks/StreamsPersistWriterTask.java   |   6 +-
 .../local/tasks/StreamsProcessorTask.java       |   2 +-
 .../local/tasks/StreamsProviderTask.java        |   5 +-
 25 files changed, 609 insertions(+), 273 deletions(-)
----------------------------------------------------------------------



[08/14] git commit: Merge remote-tracking branch 'origin/pr/7' into streamstutorial

Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/pr/7' into streamstutorial

Conflicts:
	streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
	streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/145ec847
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/145ec847
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/145ec847

Branch: refs/heads/master
Commit: 145ec8472c9cf12e8e376ce09596f354644e587b
Parents: 7293594 f62afa5
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 14:15:27 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 14:15:27 2014 -0500

----------------------------------------------------------------------
 .../ElasticsearchConfigurator.java              |   4 +-
 .../ElasticsearchPersistWriter.java             | 273 +++++++++----------
 .../ElasticsearchWriterConfiguration.json       |   5 +-
 .../provider/SysomosHeartbeatStream.java        |  45 +--
 .../local/builders/LocalStreamBuilder.java      | 161 +++++++----
 .../local/tasks/StreamsPersistWriterTask.java   |   6 +-
 .../local/tasks/StreamsProcessorTask.java       |   2 +-
 7 files changed, 271 insertions(+), 225 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/145ec847/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 9ac234c,8a343a6..e07e50f
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@@ -51,17 -51,12 +51,19 @@@ public class ElasticsearchConfigurator 
  
          String index = elasticsearch.getString("index");
          String type = elasticsearch.getString("type");
+         Long maxMsBeforeFlush = elasticsearch.hasPath("MaxTimeBetweenFlushMs") ? elasticsearch.getLong("MaxTimeBetweenFlushMs") : null;
  
 +        if( elasticsearch.hasPath("bulk"))
 +            elasticsearchWriterConfiguration.setBulk(elasticsearch.getBoolean("bulk"));
 +
 +        if( elasticsearch.hasPath("batchSize"))
 +            elasticsearchWriterConfiguration.setBatchSize(elasticsearch.getLong("batchSize"));
 +
          elasticsearchWriterConfiguration.setIndex(index);
          elasticsearchWriterConfiguration.setType(type);
+         elasticsearchWriterConfiguration.setMaxTimeBetweenFlushMs(maxMsBeforeFlush);
  
 +
          return elasticsearchWriterConfiguration;
      }
  

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/145ec847/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 80d2775,5756e1c..ce23197
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@@ -49,8 -54,11 +54,12 @@@ public class ElasticsearchPersistWrite
      private static final long WAITING_DOCS_LIMIT = 10000;
      private static final int BYTES_IN_MB = 1024 * 1024;
      private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+     private static final long DEFAULT_MAX_WAIT = 10000;
++    private static final int DEFAULT_BATCH_SIZE = 100;
  
      private final List<String> affectedIndexes = new ArrayList<String>();
+     private final ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
+     private final ReadWriteLock lock = new ReentrantReadWriteLock();
  
      private ObjectMapper mapper = new StreamsJacksonMapper();
      private ElasticsearchClientManager manager;
@@@ -59,11 -67,11 +68,10 @@@
      private String parentID = null;
      private BulkRequestBuilder bulkRequest;
      private OutputStreamWriter currentWriter = null;
 -    private int batchSize = 50;
 -    private int totalRecordsWritten = 0;
 -    private long maxMsBeforeFlush;
++    private int batchSize;
++    private long maxTimeBetweenFlushMs;
+     private boolean veryLargeBulk = false;  // by default this setting is set to false
  
-     private long batchSize;
-     private boolean veryLargeBulk;  // by default this setting is set to false
-     private int totalRecordsWritten = 0;
-     
      protected Thread task;
  
      protected volatile Queue<StreamsDatum> persistQueue;
@@@ -99,8 -106,7 +108,6 @@@
          this.veryLargeBulk = veryLargeBulk;
      }
  
-     private final List<String> affectedIndexes = new ArrayList<String>();
--
      public int getTotalOutstanding() {
          return this.totalSent - (this.totalFailed + this.totalOk);
      }
@@@ -151,6 -155,6 +156,14 @@@
          this.flushThresholdSizeInBytes = sizeInBytes;
      }
  
++    public long getMaxTimeBetweenFlushMs() {
++        return maxTimeBetweenFlushMs;
++    }
++
++    public void setMaxTimeBetweenFlushMs(long maxTimeBetweenFlushMs) {
++        this.maxTimeBetweenFlushMs = maxTimeBetweenFlushMs;
++    }
++
      public boolean isConnected() {
          return (client != null);
      }
@@@ -252,12 -250,13 +259,6 @@@
      }
  
      @Override
--    public void prepare(Object configurationObject) {
-         mapper = StreamsJacksonMapper.getInstance();
-         start();
-     }
- 
-     @Override
 -        maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? DEFAULT_MAX_WAIT : config.getMaxTimeBetweenFlushMs();
 -        mapper = StreamsJacksonMapper.getInstance();
 -        start();
 -    }
 -
 -    @Override
      public DatumStatusCounter getDatumStatusCounter() {
          DatumStatusCounter counters = new DatumStatusCounter();
          counters.incrementAttempt(this.batchItemsSent);
@@@ -267,7 -266,17 +268,17 @@@
      }
  
      public void start() {
- 
+         backgroundFlushTask.scheduleWithFixedDelay(new Runnable() {
+             @Override
+             public void run() {
+                 LOGGER.debug("Checking to see if data needs to be flushed");
+                 long time = System.currentTimeMillis() - lastWrite.get();
 -                if (time > maxMsBeforeFlush && batchItemsSent > 0) {
++                if (time > maxTimeBetweenFlushMs && batchItemsSent > 0) {
+                     LOGGER.debug("Background Flush task determined {} are waiting to be flushed.  It has been {} since the last write to ES", batchItemsSent, time);
+                     flushInternal();
+                 }
+             }
 -        }, 0, maxMsBeforeFlush * 2, TimeUnit.MILLISECONDS);
++        }, 0, maxTimeBetweenFlushMs * 2, TimeUnit.MILLISECONDS);
          manager = new ElasticsearchClientManager(config);
          client = manager.getClient();
  
@@@ -383,70 -394,20 +396,62 @@@
      }
  
      public void add(IndexRequest indexRequest) {
-         synchronized (this) {
-             checkAndCreateBulkRequest();
-             checkIndexImplications(indexRequest.index());
-             bulkRequest.add(indexRequest);
-             try {
-                 trackItemAndBytesWritten(indexRequest.source().length());
-             } catch (NullPointerException x) {
-                 LOGGER.warn("NPE adding/sizing indexrequest");
-             }
+         lock.writeLock().lock();
+         checkAndCreateBulkRequest();
+         checkIndexImplications(indexRequest.index());
+         bulkRequest.add(indexRequest);
+         try {
+             trackItemAndBytesWritten(indexRequest.source().length());
+         } catch (NullPointerException x) {
+             LOGGER.warn("NPE adding/sizing indexrequest");
+         } finally {
+             lock.writeLock().unlock();
          }
+ 
      }
  
 +    private void trackItemAndBytesWritten(long sizeInBytes)
 +    {
 +        currentItems++;
 +        batchItemsSent++;
 +        batchSizeInBytes += sizeInBytes;
 +
 +        // If our queue is larger than our flush threashold, then we should flush the queue.
 +        if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
 +                (currentItems >= batchSize) )
 +            flushInternal();
 +    }
 +
-     private void checkAndCreateBulkRequest()
-     {
-         // Synchronize to ensure that we don't lose any records
-         synchronized (this)
-         {
-             if(bulkRequest == null)
-                 bulkRequest = this.manager.getClient().prepareBulk();
-         }
-     }
- 
 +    private void checkIndexImplications(String indexName)
 +    {
 +
 +        // check to see if we have seen this index before.
 +        if(this.affectedIndexes.contains(indexName))
 +            return;
 +
 +        // we haven't log this index.
 +        this.affectedIndexes.add(indexName);
 +
 +        // Check to see if we are in 'veryLargeBulk' mode
 +        // if we aren't, exit early
 +        if(!this.veryLargeBulk)
 +            return;
 +
 +
 +        // They are in 'very large bulk' mode we want to turn off refreshing the index.
 +
 +        // Create a request then add the setting to tell it to stop refreshing the interval
 +        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
 +        updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
 +
 +        // submit to ElasticSearch
 +        this.manager.getClient()
 +                .admin()
 +                .indices()
 +                .updateSettings(updateSettingsRequest)
 +                .actionGet();
 +    }
 +
      public void createIndexIfMissing(String indexName) {
          if (!this.manager.getClient()
                  .admin()
@@@ -512,15 -473,15 +517,24 @@@
          return toReturn;
      }
  
 +    @Override
 +    public void prepare(Object configurationObject) {
 +        mapper = StreamsJacksonMapper.getInstance();
-         veryLargeBulk = this.config.getBulk();
-         batchSize = this.config.getBatchSize();
++        veryLargeBulk = config.getBulk() == null ? Boolean.FALSE : config.getBulk();
++        batchSize = config.getBatchSize() == null ? DEFAULT_BATCH_SIZE : (int)(config.getBatchSize().longValue());
++        maxTimeBetweenFlushMs = config.getMaxTimeBetweenFlushMs() == null ? DEFAULT_MAX_WAIT : config.getMaxTimeBetweenFlushMs().longValue();
 +        start();
 +    }
 +    
+     /**
+      * This method is to ONLY be called by flushInternal otherwise the counts will be off.
+      * @param bulkRequest
+      * @param thisSent
+      * @param thisSizeInBytes
+      */
      private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+         final Object messenger = new Object();
+         LOGGER.debug("Attempting to write {} items to ES", thisSent);
          bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
              @Override
              public void onResponse(BulkResponse bulkItemResponses) {
@@@ -558,18 -521,16 +574,9 @@@
                  e.printStackTrace();
              }
          });
- 
-         this.notify();
      }
  
--    private void trackItemAndBytesWritten(long sizeInBytes) {
--        batchItemsSent++;
--        batchSizeInBytes += sizeInBytes;
  
--        // If our queue is larger than our flush threashold, then we should flush the queue.
--        if (batchSizeInBytes > flushThresholdSizeInBytes)
--            flushInternal();
--    }
  
      private void checkAndCreateBulkRequest() {
          // Synchronize to ensure that we don't lose any records
@@@ -579,32 -543,32 +589,5 @@@
          }
      }
  
--    private void checkIndexImplications(String indexName) {
--
--        // check to see if we have seen this index before.
--        if (this.affectedIndexes.contains(indexName))
--            return;
--
--        // we haven't log this index.
--        this.affectedIndexes.add(indexName);
--
--        // Check to see if we are in 'veryLargeBulk' mode
--        // if we aren't, exit early
--        if (!this.veryLargeBulk)
--            return;
--
--
--        // They are in 'very large bulk' mode we want to turn off refreshing the index.
--
--        // Create a request then add the setting to tell it to stop refreshing the interval
--        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
--        updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
--
--        // submit to ElasticSearch
--        this.manager.getClient()
--                .admin()
--                .indices()
--                .updateSettings(updateSettingsRequest)
--                .actionGet();
--    }
  }
++

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/145ec847/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 5e169b7,c56aa82..b107be6
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@@ -14,15 -14,9 +14,18 @@@
              "type": "string",
              "description": "Type to write as"
          },
 -		"MaxTimeBetweenFlushMs": {
 -			"type": "String",
 -			"format": "utc-millisec"
 +        "bulk": {
 +            "type": "boolean",
 +            "description": "Index in large or small batches",
 +            "default": "false"
 +        },
 +        "batchSize": {
 +            "type": "integer",
 +            "description": "Item Count before flush",
 +            "default": 100
-         }
++        },
++		"maxTimeBetweenFlushMs": {
++			"type": "integer"
+ 		}
      }
  }


[03/14] git commit: Updated termination process

Posted by sb...@apache.org.
Updated termination process


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/6934c7a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/6934c7a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/6934c7a0

Branch: refs/heads/master
Commit: 6934c7a0d4c479392c6c147d00af8cd9ed7b3c44
Parents: c837a2c
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 18:45:30 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 18:45:30 2014 -0400

----------------------------------------------------------------------
 .../local/builders/LocalStreamBuilder.java      | 161 ++++++++++++-------
 .../local/tasks/StreamsPersistWriterTask.java   |   6 +-
 .../local/tasks/StreamsProcessorTask.java       |   2 +-
 3 files changed, 110 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6934c7a0/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
index 8e688ba..d313b3f 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/builders/LocalStreamBuilder.java
@@ -1,5 +1,6 @@
 package org.apache.streams.local.builders;
 
+import org.apache.log4j.spi.LoggerFactory;
 import org.apache.streams.core.*;
 import org.apache.streams.local.tasks.LocalStreamProcessMonitorThread;
 import org.apache.streams.local.tasks.StatusCounterMonitorThread;
@@ -7,6 +8,7 @@ import org.apache.streams.local.tasks.StreamsProviderTask;
 import org.apache.streams.local.tasks.StreamsTask;
 import org.apache.streams.util.SerializationUtil;
 import org.joda.time.DateTime;
+import org.slf4j.Logger;
 
 import java.math.BigInteger;
 import java.util.*;
@@ -22,6 +24,8 @@ import java.util.concurrent.TimeUnit;
  */
 public class LocalStreamBuilder implements StreamBuilder {
 
+    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(LocalStreamBuilder.class);
+
     public static final String TIMEOUT_KEY = "TIMEOUT";
     private Map<String, StreamComponent> providers;
     private Map<String, StreamComponent> components;
@@ -32,6 +36,7 @@ public class LocalStreamBuilder implements StreamBuilder {
     private int totalTasks;
     private int monitorTasks;
     private LocalStreamProcessMonitorThread monitorThread;
+    private Map<String, List<StreamsTask>> tasks;
 
     /**
      *
@@ -139,39 +144,18 @@ public class LocalStreamBuilder implements StreamBuilder {
      */
     @Override
     public void start() {
+        attachShutdownHandler();
         boolean isRunning = true;
         this.executor = Executors.newFixedThreadPool(this.totalTasks);
         this.monitor = Executors.newFixedThreadPool(this.monitorTasks+1);
         Map<String, StreamsProviderTask> provTasks = new HashMap<String, StreamsProviderTask>();
-        Map<String, List<StreamsTask>> streamsTasks = new HashMap<String, List<StreamsTask>>();
+        tasks = new HashMap<String, List<StreamsTask>>();
         try {
             monitorThread = new LocalStreamProcessMonitorThread(executor, 10);
             this.monitor.submit(monitorThread);
-            for(StreamComponent comp : this.components.values()) {
-                int tasks = comp.getNumTasks();
-                List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
-                for(int i=0; i < tasks; ++i) {
-                    StreamsTask task = comp.createConnectedTask(getTimeout());
-                    task.setStreamConfig(this.streamConfig);
-                    this.executor.submit(task);
-                    compTasks.add(task);
-                    if( comp.isOperationCountable() ) {
-                        this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) comp.getOperation(), 10));
-                        this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
-                    }
-                }
-                streamsTasks.put(comp.getId(), compTasks);
-            }
-            for(StreamComponent prov : this.providers.values()) {
-                StreamsTask task = prov.createConnectedTask(getTimeout());
-                task.setStreamConfig(this.streamConfig);
-                this.executor.submit(task);
-                provTasks.put(prov.getId(), (StreamsProviderTask) task);
-                if( prov.isOperationCountable() ) {
-                    this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) prov.getOperation(), 10));
-                    this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
-                }
-            }
+            setupComponentTasks(tasks);
+            setupProviderTasks(provTasks);
+            LOGGER.info("Started stream with {} components", tasks.size());
             while(isRunning) {
                 isRunning = false;
                 for(StreamsProviderTask task : provTasks.values()) {
@@ -184,44 +168,98 @@ public class LocalStreamBuilder implements StreamBuilder {
                     Thread.sleep(3000);
                 }
             }
-            monitorThread.shutdown();
-            this.executor.shutdown();
-            //complete stream shut down gracfully
-            for(StreamComponent prov : this.providers.values()) {
-                shutDownTask(prov, streamsTasks);
+            LOGGER.debug("Components are no longer running or timed out due to completion");
+            shutdown(tasks);
+        } catch (InterruptedException e){
+            forceShutdown(tasks);
+        }
+
+    }
+
+    private void attachShutdownHandler() {
+        final LocalStreamBuilder self = this;
+        LOGGER.debug("Attaching shutdown handler");
+        Runtime.getRuntime().addShutdownHook(new Thread(){
+            @Override
+            public void run() {
+                LOGGER.debug("Shutdown hook received.  Beginning shutdown");
+                self.stop();
+            }
+        });
+    }
+
+    protected void forceShutdown(Map<String, List<StreamsTask>> streamsTasks) {
+        LOGGER.debug("Shutdown failed.  Forcing shutdown");
+        //give the stream 30secs to try to shutdown gracefully, then force shutdown otherwise
+        for(List<StreamsTask> tasks : streamsTasks.values()) {
+            for(StreamsTask task : tasks) {
+                task.stopTask();
             }
-            //need to make this configurable
-            if(!this.executor.awaitTermination(10, TimeUnit.SECONDS)) { // all threads should have terminated already.
+        }
+        this.executor.shutdown();
+        this.monitor.shutdown();
+        try {
+            if(!this.executor.awaitTermination(3, TimeUnit.SECONDS)){
                 this.executor.shutdownNow();
-                this.executor.awaitTermination(10, TimeUnit.SECONDS);
             }
-            if(!this.monitor.awaitTermination(5, TimeUnit.SECONDS)) { // all threads should have terminated already.
+            if(!this.monitor.awaitTermination(3, TimeUnit.SECONDS)){
                 this.monitor.shutdownNow();
-                this.monitor.awaitTermination(5, TimeUnit.SECONDS);
             }
-        } catch (InterruptedException e){
-            //give the stream 30secs to try to shutdown gracefully, then force shutdown otherwise
-            for(List<StreamsTask> tasks : streamsTasks.values()) {
-                for(StreamsTask task : tasks) {
-                    task.stopTask();
-                }
+        }catch (InterruptedException ie) {
+            this.executor.shutdownNow();
+            this.monitor.shutdownNow();
+            throw new RuntimeException(ie);
+        }
+    }
+
+    protected void shutdown(Map<String, List<StreamsTask>> streamsTasks) throws InterruptedException {
+        LOGGER.info("Attempting to shutdown tasks");
+        monitorThread.shutdown();
+        this.executor.shutdown();
+        //complete stream shut down gracfully
+        for(StreamComponent prov : this.providers.values()) {
+            shutDownTask(prov, streamsTasks);
+        }
+        //need to make this configurable
+        if(!this.executor.awaitTermination(10, TimeUnit.SECONDS)) { // all threads should have terminated already.
+            this.executor.shutdownNow();
+            this.executor.awaitTermination(10, TimeUnit.SECONDS);
+        }
+        if(!this.monitor.awaitTermination(5, TimeUnit.SECONDS)) { // all threads should have terminated already.
+            this.monitor.shutdownNow();
+            this.monitor.awaitTermination(5, TimeUnit.SECONDS);
+        }
+    }
+
+    protected void setupProviderTasks(Map<String, StreamsProviderTask> provTasks) {
+        for(StreamComponent prov : this.providers.values()) {
+            StreamsTask task = prov.createConnectedTask(getTimeout());
+            task.setStreamConfig(this.streamConfig);
+            this.executor.submit(task);
+            provTasks.put(prov.getId(), (StreamsProviderTask) task);
+            if( prov.isOperationCountable() ) {
+                this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) prov.getOperation(), 10));
+                this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
             }
-            this.executor.shutdown();
-            this.monitor.shutdown();
-            try {
-                if(!this.executor.awaitTermination(3, TimeUnit.SECONDS)){
-                    this.executor.shutdownNow();
-                }
-                if(!this.monitor.awaitTermination(3, TimeUnit.SECONDS)){
-                    this.monitor.shutdownNow();
+        }
+    }
+
+    protected void setupComponentTasks(Map<String, List<StreamsTask>> streamsTasks) {
+        for(StreamComponent comp : this.components.values()) {
+            int tasks = comp.getNumTasks();
+            List<StreamsTask> compTasks = new LinkedList<StreamsTask>();
+            for(int i=0; i < tasks; ++i) {
+                StreamsTask task = comp.createConnectedTask(getTimeout());
+                task.setStreamConfig(this.streamConfig);
+                this.executor.submit(task);
+                compTasks.add(task);
+                if( comp.isOperationCountable() ) {
+                    this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) comp.getOperation(), 10));
+                    this.monitor.submit(new StatusCounterMonitorThread((DatumStatusCountable) task, 10));
                 }
-            }catch (InterruptedException ie) {
-                this.executor.shutdownNow();
-                this.monitor.shutdownNow();
-                throw new RuntimeException(ie);
             }
+            streamsTasks.put(comp.getId(), compTasks);
         }
-
     }
 
     /**
@@ -249,8 +287,13 @@ public class LocalStreamBuilder implements StreamBuilder {
                     task.stopTask();
                 }
                 for(StreamsTask task : tasks) {
-                    while(task.isRunning()) {
+                    int count = 0;
+                    while(count < 20 && task.isRunning()) {
                         Thread.sleep(500);
+                        count++;
+                    }
+                    if(task.isRunning()) {
+                        LOGGER.warn("Task {} failed to terminate in allotted timeframe", task.toString());
                     }
                 }
             }
@@ -268,7 +311,11 @@ public class LocalStreamBuilder implements StreamBuilder {
      */
     @Override
     public void stop() {
-
+        try {
+            shutdown(tasks);
+        } catch (Exception e) {
+            forceShutdown(tasks);
+        }
     }
 
     private void connectToOtherComponents(String[] conntectToIds, StreamComponent toBeConnected) {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6934c7a0/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
index 1eac1d9..8146bdd 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsPersistWriterTask.java
@@ -72,12 +72,13 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
         try {
             this.writer.prepare(this.streamConfig);
             StreamsDatum datum = this.inQueue.poll();
-            while(datum != null || this.keepRunning.get()) {
+            while(this.keepRunning.get()) {
                 if(datum != null) {
                     try {
                         this.writer.write(datum);
                         statusCounter.incrementStatus(DatumStatus.SUCCESS);
                     } catch (Exception e) {
+                        LOGGER.error("Error writing to persist writer {}", this.writer.getClass().getSimpleName(), e);
                         this.keepRunning.set(false);
                         statusCounter.incrementStatus(DatumStatus.FAIL);
                     }
@@ -86,12 +87,15 @@ public class StreamsPersistWriterTask extends BaseStreamsTask implements DatumSt
                     try {
                         Thread.sleep(this.sleepTime);
                     } catch (InterruptedException e) {
+                        LOGGER.warn("Thread interrupted in Writer task for {}",this.writer.getClass().getSimpleName(), e);
                         this.keepRunning.set(false);
                     }
                 }
                 datum = this.inQueue.poll();
             }
 
+        } catch(Exception e) {
+            LOGGER.error("Failed to execute Persist Writer {}",this.writer.getClass().getSimpleName(), e);
         } finally {
             this.writer.cleanUp();
             this.isRunning.set(false);

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/6934c7a0/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
----------------------------------------------------------------------
diff --git a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
index d1ac905..d4c7a16 100644
--- a/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
+++ b/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProcessorTask.java
@@ -67,7 +67,7 @@ public class StreamsProcessorTask extends BaseStreamsTask {
         try {
             this.processor.prepare(this.streamConfig);
             StreamsDatum datum = this.inQueue.poll();
-            while(datum != null || this.keepRunning.get()) {
+            while(this.keepRunning.get()) {
                 if(datum != null) {
                     List<StreamsDatum> output = this.processor.process(datum);
                     if(output != null) {


[07/14] git commit: fixes to get streams-examples working

Posted by sb...@apache.org.
fixes to get streams-examples working


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7e4e1095
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7e4e1095
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7e4e1095

Branch: refs/heads/master
Commit: 7e4e1095f09321210d6ce3acb56f4041680f2d6d
Parents: 7293594
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 13:41:28 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 13:41:28 2014 -0500

----------------------------------------------------------------------
 .../ElasticsearchPersistWriter.java             | 80 ++---------------
 .../streams-provider-twitter/pom.xml            | 13 +++
 .../provider/TwitterEventClassifier.java        | 10 ++-
 .../twitter/provider/TwitterStreamProvider.java |  7 +-
 .../provider/TwitterTimelineProvider.java       |  7 +-
 .../TwitterJsonActivitySerializer.java          |  3 +-
 .../TwitterJsonDeleteActivitySerializer.java    |  3 +-
 .../TwitterJsonRetweetActivitySerializer.java   |  3 +-
 .../TwitterJsonTweetActivitySerializer.java     |  7 +-
 ...erJsonUserstreameventActivitySerializer.java | 92 ++++++++++++++++++++
 .../main/jsonschema/com/twitter/FriendList.json | 20 +++++
 .../jsonschema/com/twitter/UserstreamEvent.json | 47 ++++++++++
 .../jackson/StreamsDateTimeDeserializer.java    |  3 +-
 .../jackson/StreamsDateTimeSerializer.java      |  5 +-
 .../jackson/StreamsPeriodDeserializer.java      |  3 +-
 .../jackson/StreamsPeriodSerializer.java        |  3 +-
 16 files changed, 211 insertions(+), 95 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 80d2775..e99c4ff 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -47,10 +47,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
     private static final Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
     private static final long WAITING_DOCS_LIMIT = 10000;
-    private static final int BYTES_IN_MB = 1024 * 1024;
-    private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
-
-    private final List<String> affectedIndexes = new ArrayList<String>();
+    private static final int  BYTES_IN_MB = 1024*1024;
+    private static final int  BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
 
     private ObjectMapper mapper = new StreamsJacksonMapper();
     private ElasticsearchClientManager manager;
@@ -62,8 +60,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     private long batchSize;
     private boolean veryLargeBulk;  // by default this setting is set to false
-    private int totalRecordsWritten = 0;
-    
+
     protected Thread task;
 
     protected volatile Queue<StreamsDatum> persistQueue;
@@ -79,8 +76,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private volatile long totalSizeInBytes = 0;
     private volatile long batchSizeInBytes = 0;
     private volatile int batchItemsSent = 0;
-    private volatile int totalByteCount = 0;
-    private volatile int byteCount = 0;
+    private volatile int  totalByteCount = 0;
+    private volatile int  byteCount = 0;
 
     public ElasticsearchPersistWriter() {
         Config config = StreamsConfigurator.config.getConfig("elasticsearch");
@@ -121,8 +118,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         return totalOk;
     }
     
-    private ObjectMapper mapper = new StreamsJacksonMapper();
-
     public int getTotalFailed() {
         return totalFailed;
     }
@@ -155,13 +150,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         return (client != null);
     }
 
-    private ElasticsearchWriterConfiguration config;
-
-    private static final int  BYTES_IN_MB = 1024*1024;
-    private static final int  BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
-    private volatile int  totalByteCount = 0;
-    private volatile int  byteCount = 0;
-    
     @Override
     public void write(StreamsDatum streamsDatum) {
 
@@ -254,6 +242,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     @Override
     public void prepare(Object configurationObject) {
         mapper = StreamsJacksonMapper.getInstance();
+        veryLargeBulk = this.config.getBulk();
+        batchSize = this.config.getBatchSize();
         start();
     }
 
@@ -395,18 +385,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         }
     }
 
-    private void trackItemAndBytesWritten(long sizeInBytes)
-    {
-        currentItems++;
-        batchItemsSent++;
-        batchSizeInBytes += sizeInBytes;
-
-        // If our queue is larger than our flush threashold, then we should flush the queue.
-        if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
-                (currentItems >= batchSize) )
-            flushInternal();
-    }
-
     private void checkAndCreateBulkRequest()
     {
         // Synchronize to ensure that we don't lose any records
@@ -512,14 +490,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         return toReturn;
     }
 
-    @Override
-    public void prepare(Object configurationObject) {
-        mapper = StreamsJacksonMapper.getInstance();
-        veryLargeBulk = this.config.getBulk();
-        batchSize = this.config.getBatchSize();
-        start();
-    }
-    
     private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
         bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
             @Override
@@ -571,40 +541,4 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
             flushInternal();
     }
 
-    private void checkAndCreateBulkRequest() {
-        // Synchronize to ensure that we don't lose any records
-        synchronized (this) {
-            if (bulkRequest == null)
-                bulkRequest = this.manager.getClient().prepareBulk();
-        }
-    }
-
-    private void checkIndexImplications(String indexName) {
-
-        // check to see if we have seen this index before.
-        if (this.affectedIndexes.contains(indexName))
-            return;
-
-        // we haven't log this index.
-        this.affectedIndexes.add(indexName);
-
-        // Check to see if we are in 'veryLargeBulk' mode
-        // if we aren't, exit early
-        if (!this.veryLargeBulk)
-            return;
-
-
-        // They are in 'very large bulk' mode we want to turn off refreshing the index.
-
-        // Create a request then add the setting to tell it to stop refreshing the interval
-        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
-        updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
-
-        // submit to ElasticSearch
-        this.manager.getClient()
-                .admin()
-                .indices()
-                .updateSettings(updateSettingsRequest)
-                .actionGet();
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/pom.xml
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/pom.xml b/streams-contrib/streams-provider-twitter/pom.xml
index 8a41ca5..c104810 100644
--- a/streams-contrib/streams-provider-twitter/pom.xml
+++ b/streams-contrib/streams-provider-twitter/pom.xml
@@ -49,6 +49,11 @@
             <artifactId>streams-config</artifactId>
         </dependency>
         <dependency>
+            <groupId>org.apache.streams</groupId>
+            <artifactId>streams-util</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
             <groupId>com.google.guava</groupId>
             <artifactId>guava</artifactId>
         </dependency>
@@ -70,6 +75,12 @@
             <artifactId>twitter4j-core</artifactId>
             <version>[4.0,)</version>
         </dependency>
+        <dependency>
+            <groupId>org.hamcrest</groupId>
+            <artifactId>hamcrest-all</artifactId>
+            <version>1.3</version>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -116,6 +127,8 @@
                         <sourcePath>src/main/jsonschema/com/twitter/TwitterStreamConfiguration.json</sourcePath>
                         <sourcePath>src/main/jsonschema/com/twitter/TwitterUserInformationConfiguration.json</sourcePath>
                         <sourcePath>src/main/jsonschema/com/twitter/Delete.json</sourcePath>
+                        <sourcePath>src/main/jsonschema/com/twitter/UserstreamEvent.json</sourcePath>
+                        <sourcePath>src/main/jsonschema/com/twitter/FriendList.json</sourcePath>
                         <sourcePath>src/main/jsonschema/com/twitter/Retweet.json</sourcePath>
                         <sourcePath>src/main/jsonschema/com/twitter/tweet.json</sourcePath>
                     </sourcePaths>

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
index b577e42..3a8bf6d 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterEventClassifier.java
@@ -2,11 +2,8 @@ package org.apache.streams.twitter.provider;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Preconditions;
-import com.jayway.jsonassert.JsonAssert;
 import org.apache.commons.lang.StringUtils;
-import org.apache.streams.twitter.pojo.Delete;
-import org.apache.streams.twitter.pojo.Retweet;
-import org.apache.streams.twitter.pojo.Tweet;
+import org.apache.streams.twitter.pojo.*;
 import org.apache.streams.twitter.serializer.StreamsTwitterMapper;
 
 import java.io.IOException;
@@ -47,6 +44,11 @@ public class TwitterEventClassifier {
             return Retweet.class;
         else if( objectNode.findValue("delete") != null )
             return Delete.class;
+        else if( objectNode.findValue("friends") != null ||
+                 objectNode.findValue("friends_str") != null )
+            return FriendList.class;
+        else if( objectNode.findValue("target_object") != null )
+            return UserstreamEvent.class;
         else
             return Tweet.class;
     }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
index ba88803..fb160ef 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterStreamProvider.java
@@ -50,9 +50,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable, Dat
         this.config = config;
     }
 
-    protected BlockingQueue hosebirdQueue = new LinkedBlockingQueue<String>(1000);
+    protected BlockingQueue<String> hosebirdQueue;
 
-    protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>(1000);
+    protected volatile Queue<StreamsDatum> providerQueue;
 
     protected Hosts hosebirdHosts;
     protected Authentication auth;
@@ -194,6 +194,9 @@ public class TwitterStreamProvider implements StreamsProvider, Serializable, Dat
 
         LOGGER.debug("host={}\tendpoint={}\taut={}", new Object[] {hosebirdHosts,endpoint,auth});
 
+        hosebirdQueue = new LinkedBlockingQueue<String>(1000);
+        providerQueue = new LinkedBlockingQueue<StreamsDatum>(1000);
+
         client = new ClientBuilder()
             .name("apache/streams/streams-contrib/streams-provider-twitter")
             .hosts(hosebirdHosts)

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index db1ec76..06ba7be 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -23,9 +23,7 @@ import twitter4j.json.DataObjectFactory;
 
 import java.io.Serializable;
 import java.math.BigInteger;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Queue;
+import java.util.*;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -260,7 +258,8 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
                 .setAsyncNumThreads(3)
                 .setRestBaseURL(baseUrl)
                 .setIncludeMyRetweetEnabled(Boolean.TRUE)
-                .setIncludeRTsEnabled(Boolean.TRUE)
+                // not sure where this method went...
+                //.setIncludeRTsEnabled(Boolean.TRUE)
                 .setPrettyDebugEnabled(Boolean.TRUE);
 
         return new TwitterFactory(builder.build()).getInstance();

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
index bfceae0..0ab3448 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonActivitySerializer.java
@@ -31,13 +31,14 @@ import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.List;
 import java.util.Map;
 
 /**
  * Created by sblackmon on 3/26/14.
  */
-public class TwitterJsonActivitySerializer implements ActivitySerializer<String>
+public class TwitterJsonActivitySerializer implements ActivitySerializer<String>, Serializable
 {
 
     public TwitterJsonActivitySerializer() {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
index 40be0f6..4a0f348 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonDeleteActivitySerializer.java
@@ -13,6 +13,7 @@ import org.apache.streams.pojo.json.Actor;
 import org.apache.streams.twitter.pojo.Delete;
 import org.apache.streams.twitter.pojo.Tweet;
 
+import java.io.Serializable;
 import java.util.List;
 
 import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerializer.*;
@@ -24,7 +25,7 @@ import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerialize
 * Time: 9:24 AM
 * To change this template use File | Settings | File Templates.
 */
-public class TwitterJsonDeleteActivitySerializer implements ActivitySerializer<String> {
+public class TwitterJsonDeleteActivitySerializer implements ActivitySerializer<String>, Serializable {
 
     @Override
     public String serializationFormat() {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
index 51e39b1..f8ade3f 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonRetweetActivitySerializer.java
@@ -17,6 +17,7 @@ import org.apache.streams.twitter.pojo.Tweet;
 import org.apache.streams.twitter.pojo.User;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -31,7 +32,7 @@ import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
 * Time: 9:24 AM
 * To change this template use File | Settings | File Templates.
 */
-public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<String> {
+public class TwitterJsonRetweetActivitySerializer implements ActivitySerializer<String>, Serializable {
 
     public TwitterJsonRetweetActivitySerializer() {
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
index b141482..1d00b7a 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonTweetActivitySerializer.java
@@ -18,6 +18,7 @@ import org.apache.streams.twitter.pojo.Tweet;
 import org.apache.streams.twitter.pojo.User;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -32,11 +33,7 @@ import static org.apache.streams.data.util.ActivityUtil.ensureExtensions;
 * Time: 9:24 AM
 * To change this template use File | Settings | File Templates.
 */
-public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<String> {
-
-    public TwitterJsonTweetActivitySerializer() {
-
-    }
+public class TwitterJsonTweetActivitySerializer implements ActivitySerializer<String>, Serializable {
 
     @Override
     public String serializationFormat() {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java
new file mode 100644
index 0000000..c0768e3
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/serializer/TwitterJsonUserstreameventActivitySerializer.java
@@ -0,0 +1,92 @@
+package org.apache.streams.twitter.serializer;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Strings;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.streams.data.ActivitySerializer;
+import org.apache.streams.exceptions.ActivitySerializerException;
+import org.apache.streams.pojo.json.Activity;
+import org.apache.streams.pojo.json.ActivityObject;
+import org.apache.streams.pojo.json.Actor;
+import org.apache.streams.twitter.pojo.Delete;
+import org.apache.streams.twitter.pojo.Tweet;
+import org.apache.streams.twitter.pojo.UserstreamEvent;
+
+import java.util.List;
+
+import static org.apache.streams.twitter.serializer.TwitterJsonActivitySerializer.*;
+
+/**
+* Created with IntelliJ IDEA.
+* User: mdelaet
+* Date: 9/30/13
+* Time: 9:24 AM
+* To change this template use File | Settings | File Templates.
+*/
+public class TwitterJsonUserstreameventActivitySerializer implements ActivitySerializer<String> {
+
+    @Override
+    public String serializationFormat() {
+        return null;
+    }
+
+    @Override
+    public String serialize(Activity deserialized) throws ActivitySerializerException {
+        throw new NotImplementedException();
+    }
+
+    @Override
+    public Activity deserialize(String serialized) throws ActivitySerializerException {
+        return null;
+    }
+
+    @Override
+    public List<Activity> deserializeAll(List<String> serializedList) {
+        return null;
+    }
+
+    public Activity convert(ObjectNode item) throws ActivitySerializerException {
+
+        ObjectMapper mapper = StreamsTwitterMapper.getInstance();
+        UserstreamEvent event = null;
+        try {
+            event = mapper.treeToValue(item, UserstreamEvent.class);
+        } catch (JsonProcessingException e) {
+            e.printStackTrace();
+        }
+
+        Activity activity = new Activity();
+        activity.setActor(buildActor(event));
+        activity.setVerb(detectVerb(event));
+        activity.setObject(buildActivityObject(event));
+        activity.setId(TwitterJsonActivitySerializer.formatId(activity.getVerb()));
+        if(Strings.isNullOrEmpty(activity.getId()))
+            throw new ActivitySerializerException("Unable to determine activity id");
+        activity.setProvider(getProvider());
+        return activity;
+    }
+
+    public Actor buildActor(UserstreamEvent event) {
+        Actor actor = new Actor();
+        //actor.setId(formatId(delete.getDelete().getStatus().getUserIdStr()));
+        return actor;
+    }
+
+    public ActivityObject buildActivityObject(UserstreamEvent event) {
+        ActivityObject actObj = new ActivityObject();
+        //actObj.setId(formatId(delete.getDelete().getStatus().getIdStr()));
+        //actObj.setObjectType("tweet");
+        return actObj;
+    }
+
+    public String detectVerb(UserstreamEvent event) {
+        return null;
+    }
+
+    public ActivityObject buildTarget(UserstreamEvent event) {
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
new file mode 100644
index 0000000..5dd7687
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/FriendList.json
@@ -0,0 +1,20 @@
+{
+    "type": "object",
+    "$schema": "http://json-schema.org/draft-03/schema",
+    "id": "#",
+    "javaType" : "org.apache.streams.twitter.pojo.FriendList",
+    "properties": {
+        "friends": {
+            "type": "array",
+            "items": {
+                "type": "integer"
+            }
+        },
+        "friends_str": {
+            "type": "array",
+            "items": {
+                "type": "string"
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
new file mode 100644
index 0000000..07b5883
--- /dev/null
+++ b/streams-contrib/streams-provider-twitter/src/main/jsonschema/com/twitter/UserstreamEvent.json
@@ -0,0 +1,47 @@
+{
+    "type": "object",
+    "$schema": "http://json-schema.org/draft-03/schema",
+    "id": "#",
+    "javaType" : "org.apache.streams.twitter.pojo.UserstreamEvent",
+    "properties": {
+        "created_at": {
+            "type": "string",
+            "format" : "date-time"
+        },
+        "event_type": {
+            "type": "string",
+            "enum" : [
+                "access_revoked",
+                "block",
+                "unblock",
+                "favorite",
+                "unfavorite",
+                "follow",
+                "unfollow",
+                "list_created",
+                "list_destroyed",
+                "list_updated",
+                "list_member_added",
+                "list_member_removed",
+                "list_user_subscribed",
+                "list_user_unsubscribed",
+                "user_update"
+            ]
+        },
+        "source": {
+            "type": "string",
+            "items": {
+                "type": "integer"
+            }
+        },
+        "target": {
+            "type": "string",
+            "items": {
+                "type": "integer"
+            }
+        },
+        "target_object": {
+            "type": "object"
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
index 3286e74..28f297c 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeDeserializer.java
@@ -7,11 +7,12 @@ import org.apache.streams.data.util.RFC3339Utils;
 import org.joda.time.DateTime;
 
 import java.io.IOException;
+import java.io.Serializable;
 
 /**
  * Created by sblackmon on 3/27/14.
  */
-public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> {
+public class StreamsDateTimeDeserializer extends StdDeserializer<DateTime> implements Serializable {
 
     protected StreamsDateTimeDeserializer(Class<DateTime> dateTimeClass) {
         super(dateTimeClass);

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
index 26bc157..89f9100 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsDateTimeSerializer.java
@@ -11,11 +11,14 @@ import org.joda.time.format.DateTimeFormatter;
 import org.joda.time.format.ISODateTimeFormat;
 
 import java.io.IOException;
+import java.io.Serializable;
 
 /**
  * Created by sblackmon on 3/27/14.
  */
-public class StreamsDateTimeSerializer extends StdSerializer<DateTime> {
+public class StreamsDateTimeSerializer extends StdSerializer<DateTime> implements Serializable {
+
+
 
     protected StreamsDateTimeSerializer(Class<DateTime> dateTimeClass) {
         super(dateTimeClass);

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
index ff765f6..9402878 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodDeserializer.java
@@ -7,8 +7,9 @@ import org.apache.streams.data.util.RFC3339Utils;
 import org.joda.time.Period;
 
 import java.io.IOException;
+import java.io.Serializable;
 
-public class StreamsPeriodDeserializer extends StdDeserializer<Period>
+public class StreamsPeriodDeserializer extends StdDeserializer<Period> implements Serializable
 {
 
     protected StreamsPeriodDeserializer(Class<Period> dateTimeClass) {

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7e4e1095/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
----------------------------------------------------------------------
diff --git a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
index 614cbdd..ef28c8e 100644
--- a/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
+++ b/streams-pojo/src/main/java/org/apache/streams/jackson/StreamsPeriodSerializer.java
@@ -8,8 +8,9 @@ import org.joda.time.DateTime;
 import org.joda.time.Period;
 
 import java.io.IOException;
+import java.io.Serializable;
 
-public class StreamsPeriodSerializer extends StdSerializer<Period>
+public class StreamsPeriodSerializer extends StdSerializer<Period> implements Serializable
 {
     protected StreamsPeriodSerializer(Class<Period> dateTimeClass) {
         super(dateTimeClass);


[06/14] git commit: merging fixes and addressing serialization problems to get twitter streams-examples working.

Posted by sb...@apache.org.
merging fixes and addressing serialization problems to get twitter streams-examples working.


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/7293594e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/7293594e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/7293594e

Branch: refs/heads/master
Commit: 7293594eac98535e4b2206dca68da9b73e6cdf24
Parents: ae27541 522096e
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 12:56:36 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 12:56:36 2014 -0500

----------------------------------------------------------------------
 .../streams-persist-elasticsearch/pom.xml       |  1 -
 .../ElasticsearchConfigurator.java              |  7 ++
 .../ElasticsearchPersistWriter.java             | 80 +++++++++++++++++++-
 .../ElasticsearchWriterConfiguration.json       | 10 +++
 .../twitter/processor/TwitterTypeConverter.java |  2 +-
 .../twitter/provider/TwitterStreamProvider.java | 68 ++++++++---------
 .../provider/TwitterTimelineProvider.java       | 50 ++++++++----
 .../local/tasks/StreamsProviderTask.java        |  2 +-
 8 files changed, 161 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ff7e0f5,405d1b8..80d2775
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@@ -59,96 -50,68 +59,109 @@@ public class ElasticsearchPersistWrite
      private String parentID = null;
      private BulkRequestBuilder bulkRequest;
      private OutputStreamWriter currentWriter = null;
-     private int batchSize = 50;
-     private int totalRecordsWritten = 0;
-     private boolean veryLargeBulk = false;  // by default this setting is set to false
  
+     private long batchSize;
+     private boolean veryLargeBulk;  // by default this setting is set to false
++    private int totalRecordsWritten = 0;
++    
 +    protected Thread task;
  
 -    private final static Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
 -    private static final long WAITING_DOCS_LIMIT = 10000;
 -
 -    public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
 +    protected volatile Queue<StreamsDatum> persistQueue;
  
+     private volatile int currentItems = 0;
      private volatile int totalSent = 0;
      private volatile int totalSeconds = 0;
      private volatile int totalAttempted = 0;
      private volatile int totalOk = 0;
      private volatile int totalFailed = 0;
      private volatile int totalBatchCount = 0;
+     private volatile int totalRecordsWritten = 0;
      private volatile long totalSizeInBytes = 0;
 -
      private volatile long batchSizeInBytes = 0;
      private volatile int batchItemsSent = 0;
 +    private volatile int totalByteCount = 0;
 +    private volatile int byteCount = 0;
 +
 +    public ElasticsearchPersistWriter() {
 +        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
 +        this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
 +    }
 +
 +    public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
 +        this.config = config;
 +    }
 +
 +    public void setBatchSize(int batchSize) {
 +        this.batchSize = batchSize;
 +    }
 +
 +    public void setVeryLargeBulk(boolean veryLargeBulk) {
 +        this.veryLargeBulk = veryLargeBulk;
 +    }
  
+     private final List<String> affectedIndexes = new ArrayList<String>();
  
 -    public int getTotalOutstanding()                           { return this.totalSent - (this.totalFailed + this.totalOk); }
 -    public long getFlushThresholdSizeInBytes()                 { return flushThresholdSizeInBytes; }
 -    public int getTotalSent()                                  { return totalSent; }
 -    public int getTotalSeconds()                               { return totalSeconds; }
 -    public int getTotalOk()                                    { return totalOk; }
 -    public int getTotalFailed()                                { return totalFailed; }
 -    public int getTotalBatchCount()                            { return totalBatchCount; }
 -    public long getTotalSizeInBytes()                          { return totalSizeInBytes; }
 -    public long getBatchSizeInBytes()                          { return batchSizeInBytes; }
 -    public int getBatchItemsSent()                             { return batchItemsSent; }
 -    public List<String> getAffectedIndexes()                   { return this.affectedIndexes; }
 +    public int getTotalOutstanding() {
 +        return this.totalSent - (this.totalFailed + this.totalOk);
 +    }
  
 -    public void setFlushThresholdSizeInBytes(long sizeInBytes)  { this.flushThresholdSizeInBytes = sizeInBytes; }
 +    public long getFlushThresholdSizeInBytes() {
 +        return flushThresholdSizeInBytes;
 +    }
  
 -    Thread task;
 +    public int getTotalSent() {
 +        return totalSent;
 +    }
  
 -    protected volatile Queue<StreamsDatum> persistQueue;
 +    public int getTotalSeconds() {
 +        return totalSeconds;
 +    }
  
 -    private ObjectMapper mapper;
 +    public int getTotalOk() {
 +        return totalOk;
 +    }
++    
++    private ObjectMapper mapper = new StreamsJacksonMapper();
  
 -    private ElasticsearchWriterConfiguration config;
 +    public int getTotalFailed() {
 +        return totalFailed;
 +    }
  
 -    public ElasticsearchPersistWriter() {
 -        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
 -        this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
 +    public int getTotalBatchCount() {
 +        return totalBatchCount;
      }
  
 -    public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
 -        this.config = config;
 +    public long getTotalSizeInBytes() {
 +        return totalSizeInBytes;
 +    }
 +
 +    public long getBatchSizeInBytes() {
 +        return batchSizeInBytes;
 +    }
 +
 +    public int getBatchItemsSent() {
 +        return batchItemsSent;
 +    }
 +
 +    public List<String> getAffectedIndexes() {
 +        return this.affectedIndexes;
 +    }
 +
 +    public void setFlushThresholdSizeInBytes(long sizeInBytes) {
 +        this.flushThresholdSizeInBytes = sizeInBytes;
      }
  
 +    public boolean isConnected() {
 +        return (client != null);
 +    }
 +
++    private ElasticsearchWriterConfiguration config;
++
+     private static final int  BYTES_IN_MB = 1024*1024;
+     private static final int  BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+     private volatile int  totalByteCount = 0;
+     private volatile int  byteCount = 0;
 -
 -    public boolean isConnected() 		                { return (client != null); }
 -
++    
      @Override
      public void write(StreamsDatum streamsDatum) {
  
@@@ -280,10 -238,13 +293,11 @@@
              // reset the current batch statistics
              this.batchSizeInBytes = 0;
              this.batchItemsSent = 0;
+             this.currentItems = 0;
  
 -            try
 -            {
 +            try {
                  int count = 0;
 -                if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
 -                {
 +                if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
                      /****************************************************************************
                       * Author:
                       * Smashew
@@@ -381,8 -402,60 +395,60 @@@
          }
      }
  
+     private void trackItemAndBytesWritten(long sizeInBytes)
+     {
+         currentItems++;
+         batchItemsSent++;
+         batchSizeInBytes += sizeInBytes;
+ 
+         // If our queue is larger than our flush threashold, then we should flush the queue.
+         if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
+                 (currentItems >= batchSize) )
+             flushInternal();
+     }
+ 
+     private void checkAndCreateBulkRequest()
+     {
+         // Synchronize to ensure that we don't lose any records
+         synchronized (this)
+         {
+             if(bulkRequest == null)
+                 bulkRequest = this.manager.getClient().prepareBulk();
+         }
+     }
+ 
+     private void checkIndexImplications(String indexName)
+     {
+ 
+         // check to see if we have seen this index before.
+         if(this.affectedIndexes.contains(indexName))
+             return;
+ 
+         // we haven't log this index.
+         this.affectedIndexes.add(indexName);
+ 
+         // Check to see if we are in 'veryLargeBulk' mode
+         // if we aren't, exit early
+         if(!this.veryLargeBulk)
+             return;
+ 
+ 
+         // They are in 'very large bulk' mode we want to turn off refreshing the index.
+ 
+         // Create a request then add the setting to tell it to stop refreshing the interval
+         UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
+         updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
+ 
+         // submit to ElasticSearch
+         this.manager.getClient()
+                 .admin()
+                 .indices()
+                 .updateSettings(updateSettingsRequest)
+                 .actionGet();
+     }
+ 
      public void createIndexIfMissing(String indexName) {
 -        if(!this.manager.getClient()
 +        if (!this.manager.getClient()
                  .admin()
                  .indices()
                  .exists(new IndicesExistsRequest(indexName))
@@@ -446,91 -524,20 +512,99 @@@
          return toReturn;
      }
  
+     @Override
+     public void prepare(Object configurationObject) {
+         mapper = StreamsJacksonMapper.getInstance();
+         veryLargeBulk = this.config.getBulk();
+         batchSize = this.config.getBatchSize();
+         start();
+     }
++    
 +    private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
 +        bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
 +            @Override
 +            public void onResponse(BulkResponse bulkItemResponses) {
 +                if (bulkItemResponses.hasFailures())
 +                    LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
  
 -    @Override
 -    public DatumStatusCounter getDatumStatusCounter() {
 -        DatumStatusCounter counters = new DatumStatusCounter();
 -        counters.incrementAttempt(this.batchItemsSent);
 -        counters.incrementStatus(DatumStatus.SUCCESS, this.totalOk);
 -        counters.incrementStatus(DatumStatus.FAIL, this.totalFailed);
 -        return counters;
 +                long thisFailed = 0;
 +                long thisOk = 0;
 +                long thisMillis = bulkItemResponses.getTookInMillis();
 +
 +                // keep track of the number of totalFailed and items that we have totalOk.
 +                for (BulkItemResponse resp : bulkItemResponses.getItems()) {
 +                    if (resp.isFailed())
 +                        thisFailed++;
 +                    else
 +                        thisOk++;
 +                }
 +
 +                totalAttempted += thisSent;
 +                totalOk += thisOk;
 +                totalFailed += thisFailed;
 +                totalSeconds += (thisMillis / 1000);
 +
 +                if (thisSent != (thisOk + thisFailed))
 +                    LOGGER.error("We sent more items than this");
 +
 +                LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
 +                        MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
 +                        MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
 +            }
 +
 +            @Override
 +            public void onFailure(Throwable e) {
 +                LOGGER.error("Error bulk loading: {}", e.getMessage());
 +                e.printStackTrace();
 +            }
 +        });
 +
 +        this.notify();
 +    }
 +
 +    private void trackItemAndBytesWritten(long sizeInBytes) {
 +        batchItemsSent++;
 +        batchSizeInBytes += sizeInBytes;
 +
 +        // If our queue is larger than our flush threashold, then we should flush the queue.
 +        if (batchSizeInBytes > flushThresholdSizeInBytes)
 +            flushInternal();
 +    }
 +
 +    private void checkAndCreateBulkRequest() {
 +        // Synchronize to ensure that we don't lose any records
 +        synchronized (this) {
 +            if (bulkRequest == null)
 +                bulkRequest = this.manager.getClient().prepareBulk();
 +        }
 +    }
 +
 +    private void checkIndexImplications(String indexName) {
 +
 +        // check to see if we have seen this index before.
 +        if (this.affectedIndexes.contains(indexName))
 +            return;
 +
 +        // we haven't log this index.
 +        this.affectedIndexes.add(indexName);
 +
 +        // Check to see if we are in 'veryLargeBulk' mode
 +        // if we aren't, exit early
 +        if (!this.veryLargeBulk)
 +            return;
 +
 +
 +        // They are in 'very large bulk' mode we want to turn off refreshing the index.
 +
 +        // Create a request then add the setting to tell it to stop refreshing the interval
 +        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
 +        updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
 +
 +        // submit to ElasticSearch
 +        this.manager.getClient()
 +                .admin()
 +                .indices()
 +                .updateSettings(updateSettingsRequest)
 +                .actionGet();
      }
  }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --cc streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index b456fa4,20ee951..db1ec76
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@@ -1,7 -1,8 +1,9 @@@
  package org.apache.streams.twitter.provider;
  
+ import com.google.common.base.Optional;
  import com.google.common.base.Preconditions;
 +import com.google.common.collect.Queues;
+ import com.google.common.util.concurrent.ListenableFuture;
  import com.google.common.util.concurrent.ListeningExecutorService;
  import com.google.common.util.concurrent.MoreExecutors;
  import com.typesafe.config.Config;
@@@ -20,11 -21,10 +23,13 @@@ import twitter4j.json.DataObjectFactory
  
  import java.io.Serializable;
  import java.math.BigInteger;
- import java.util.*;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Queue;
 -import java.util.concurrent.*;
 +import java.util.concurrent.ArrayBlockingQueue;
 +import java.util.concurrent.ExecutorService;
 +import java.util.concurrent.ThreadPoolExecutor;
 +import java.util.concurrent.TimeUnit;
  
  /**
   * Created by sblackmon on 12/10/13.
@@@ -47,16 -47,22 +52,20 @@@ public class TwitterTimelineProvider im
          this.config = config;
      }
  
 -    protected volatile Queue<StreamsDatum> providerQueue = new LinkedBlockingQueue<StreamsDatum>();
 +    protected final Queue<StreamsDatum> providerQueue = Queues.synchronizedQueue(new ArrayBlockingQueue<StreamsDatum>(500));
  
 +    protected int idsCount;
+     protected Twitter client;
      protected Iterator<Long> ids;
  
 -    ListenableFuture providerTaskComplete;
 -//
 -//    public BlockingQueue<Object> getInQueue() {
 -//        return inQueue;
 -//    }
 -
      protected ListeningExecutorService executor;
  
      protected DateTime start;
      protected DateTime end;
  
++    Boolean jsonStoreEnabled;
++    Boolean includeEntitiesEnabled;
++
      private static ExecutorService newFixedThreadPoolWithQueueSize(int nThreads, int queueSize) {
          return new ThreadPoolExecutor(nThreads, nThreads,
                  5000L, TimeUnit.MILLISECONDS,
@@@ -88,12 -93,36 +96,13 @@@
          return this.providerQueue;
      }
  
 -//    public void run() {
 -//
 -//        LOGGER.info("{} Running", STREAMS_ID);
 -//
 -//        while( ids.hasNext() ) {
 -//            Long currentId = ids.next();
 -//            LOGGER.info("Provider Task Starting: {}", currentId);
 -//            captureTimeline(currentId);
 -//        }
 -//
 -//        LOGGER.info("{} Finished.  Cleaning up...", STREAMS_ID);
 -//
 -//        client.shutdown();
 -//
 -//        LOGGER.info("{} Exiting", STREAMS_ID);
 -//
 -//        while(!providerTaskComplete.isDone() && !providerTaskComplete.isCancelled() ) {
 -//            try {
 -//                Thread.sleep(100);
 -//            } catch (InterruptedException e) {}
 -//        }
 -//    }
 -
      @Override
      public void startStream() {
-         // no op
+         LOGGER.debug("{} startStream", STREAMS_ID);
+         throw new org.apache.commons.lang.NotImplementedException();
      }
  
 -    private void captureTimeline(long currentId) {
 +    protected void captureTimeline(long currentId) {
  
          Paging paging = new Paging(1, 200);
          List<Status> statuses = null;
@@@ -223,14 -251,14 +235,19 @@@
          Preconditions.checkNotNull(config.getOauth().getConsumerSecret());
          Preconditions.checkNotNull(config.getOauth().getAccessToken());
          Preconditions.checkNotNull(config.getOauth().getAccessTokenSecret());
 -
          Preconditions.checkNotNull(config.getFollow());
  
 -        Boolean jsonStoreEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getJsonStoreEnabled()))).or(true);
 -        Boolean includeEntitiesEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getIncludeEntities()))).or(true);
 -
 +        idsCount = config.getFollow().size();
          ids = config.getFollow().iterator();
+ 
++        jsonStoreEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getJsonStoreEnabled()))).or(true);
++        includeEntitiesEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getIncludeEntities()))).or(true);
++
 +    }
 +
++    
 +    protected Twitter getTwitterClient()
 +    {
          String baseUrl = "https://api.twitter.com:443/1.1/";
  
          ConfigurationBuilder builder = new ConfigurationBuilder()
@@@ -243,9 -271,11 +260,10 @@@
                  .setAsyncNumThreads(3)
                  .setRestBaseURL(baseUrl)
                  .setIncludeMyRetweetEnabled(Boolean.TRUE)
+                 .setIncludeRTsEnabled(Boolean.TRUE)
                  .setPrettyDebugEnabled(Boolean.TRUE);
  
 -        client = new TwitterFactory(builder.build()).getInstance();
 -
 +        return new TwitterFactory(builder.build()).getInstance();
      }
  
      @Override

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/7293594e/streams-runtimes/streams-runtime-local/src/main/java/org/apache/streams/local/tasks/StreamsProviderTask.java
----------------------------------------------------------------------


[04/14] git commit: Fixed deadlock issue

Posted by sb...@apache.org.
Fixed deadlock issue


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/fb8f7aea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/fb8f7aea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/fb8f7aea

Branch: refs/heads/master
Commit: fb8f7aea3032296a4c0dd779aca976104de44d61
Parents: 6934c7a
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 19:53:18 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 19:53:18 2014 -0400

----------------------------------------------------------------------
 .../ElasticsearchPersistWriter.java             | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/fb8f7aea/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index c574eeb..5756e1c 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -54,7 +54,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private static final long WAITING_DOCS_LIMIT = 10000;
     private static final int BYTES_IN_MB = 1024 * 1024;
     private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
-    private static final long MAX_MS_BEFORE_FLUSH = 10000;
+    private static final long DEFAULT_MAX_WAIT = 10000;
 
     private final List<String> affectedIndexes = new ArrayList<String>();
     private final ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
@@ -251,7 +251,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     @Override
     public void prepare(Object configurationObject) {
-        maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? MAX_MS_BEFORE_FLUSH : config.getMaxTimeBetweenFlushMs();
+        maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? DEFAULT_MAX_WAIT : config.getMaxTimeBetweenFlushMs();
         mapper = StreamsJacksonMapper.getInstance();
         start();
     }
@@ -284,7 +284,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     }
 
     public void flushInternal() {
-        //This is unlocked in the callback
         lock.writeLock().lock();
         // we do not have a working bulk request, we can just exit here.
         if (this.bulkRequest == null || batchItemsSent == 0)
@@ -339,6 +338,8 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
             }
         } catch (Exception e) {
             LOGGER.info("We were broken from our loop: {}", e.getMessage());
+        } finally {
+            lock.writeLock().unlock();
         }
 
     }
@@ -472,11 +473,20 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         return toReturn;
     }
 
+    /**
+     * This method is to ONLY be called by flushInternal otherwise the counts will be off.
+     * @param bulkRequest
+     * @param thisSent
+     * @param thisSizeInBytes
+     */
     private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+        final Object messenger = new Object();
         LOGGER.debug("Attempting to write {} items to ES", thisSent);
         bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
             @Override
             public void onResponse(BulkResponse bulkItemResponses) {
+                lastWrite.set(System.currentTimeMillis());
+
                 if (bulkItemResponses.hasFailures())
                     LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
 
@@ -500,9 +510,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
                 if (thisSent != (thisOk + thisFailed))
                     LOGGER.error("We sent more items than this");
 
-                lastWrite.set(System.currentTimeMillis());
-                lock.writeLock().unlock();
-
                 LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
                         MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
                         MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
@@ -510,7 +517,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
             @Override
             public void onFailure(Throwable e) {
-                lock.writeLock().unlock();
                 LOGGER.error("Error bulk loading: {}", e.getMessage());
                 e.printStackTrace();
             }


[02/14] git commit: Updated Persist writer to have a configurable background flush thread

Posted by sb...@apache.org.
Updated Persist writer to have a configurable background flush thread


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/c837a2cf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/c837a2cf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/c837a2cf

Branch: refs/heads/master
Commit: c837a2cf8800efd324003ba5a9299c16b9f8701a
Parents: e6ffe29
Author: mfranklin <mf...@apache.org>
Authored: Thu May 1 13:59:32 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Thu May 1 13:59:32 2014 -0400

----------------------------------------------------------------------
 .../ElasticsearchConfigurator.java              |   4 +-
 .../ElasticsearchPersistWriter.java             | 184 +++++++++++--------
 .../ElasticsearchWriterConfiguration.json       |   6 +-
 3 files changed, 116 insertions(+), 78 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c837a2cf/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 483a177..8a343a6 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@ -8,7 +8,7 @@ import org.slf4j.LoggerFactory;
 import java.util.List;
 
 /**
- * Created by sblackmon on 12/10/13.
+ * Converts a {@link com.typesafe.config.Config} element into an instance of ElasticSearchConfiguration
  */
 public class ElasticsearchConfigurator {
 
@@ -51,9 +51,11 @@ public class ElasticsearchConfigurator {
 
         String index = elasticsearch.getString("index");
         String type = elasticsearch.getString("type");
+        Long maxMsBeforeFlush = elasticsearch.hasPath("MaxTimeBetweenFlushMs") ? elasticsearch.getLong("MaxTimeBetweenFlushMs") : null;
 
         elasticsearchWriterConfiguration.setIndex(index);
         elasticsearchWriterConfiguration.setType(type);
+        elasticsearchWriterConfiguration.setMaxTimeBetweenFlushMs(maxMsBeforeFlush);
 
         return elasticsearchWriterConfiguration;
     }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c837a2cf/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ff7e0f5..c574eeb 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -36,10 +36,15 @@ import java.io.OutputStreamWriter;
 import java.text.DecimalFormat;
 import java.text.NumberFormat;
 import java.util.*;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable {
     public static final String STREAMS_ID = "ElasticsearchPersistWriter";
-
     public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
@@ -49,8 +54,11 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private static final long WAITING_DOCS_LIMIT = 10000;
     private static final int BYTES_IN_MB = 1024 * 1024;
     private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+    private static final long MAX_MS_BEFORE_FLUSH = 10000;
 
     private final List<String> affectedIndexes = new ArrayList<String>();
+    private final ScheduledExecutorService backgroundFlushTask = Executors.newSingleThreadScheduledExecutor();
+    private final ReadWriteLock lock = new ReentrantReadWriteLock();
 
     private ObjectMapper mapper = new StreamsJacksonMapper();
     private ElasticsearchClientManager manager;
@@ -61,6 +69,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private OutputStreamWriter currentWriter = null;
     private int batchSize = 50;
     private int totalRecordsWritten = 0;
+    private long maxMsBeforeFlush;
     private boolean veryLargeBulk = false;  // by default this setting is set to false
 
     protected Thread task;
@@ -78,10 +87,11 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private volatile int batchItemsSent = 0;
     private volatile int totalByteCount = 0;
     private volatile int byteCount = 0;
+    private volatile AtomicLong lastWrite = new AtomicLong(System.currentTimeMillis());
 
     public ElasticsearchPersistWriter() {
         Config config = StreamsConfigurator.config.getConfig("elasticsearch");
-        this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+        this.config = ElasticsearchConfigurator.detectWriterConfiguration(config);
     }
 
     public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
@@ -173,6 +183,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
         try {
             flush();
+            backgroundFlushTask.shutdownNow();
         } catch (IOException e) {
             e.printStackTrace();
         }
@@ -240,6 +251,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     @Override
     public void prepare(Object configurationObject) {
+        maxMsBeforeFlush = config.getMaxTimeBetweenFlushMs() == null ? MAX_MS_BEFORE_FLUSH : config.getMaxTimeBetweenFlushMs();
         mapper = StreamsJacksonMapper.getInstance();
         start();
     }
@@ -254,7 +266,17 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     }
 
     public void start() {
-
+        backgroundFlushTask.scheduleWithFixedDelay(new Runnable() {
+            @Override
+            public void run() {
+                LOGGER.debug("Checking to see if data needs to be flushed");
+                long time = System.currentTimeMillis() - lastWrite.get();
+                if (time > maxMsBeforeFlush && batchItemsSent > 0) {
+                    LOGGER.debug("Background Flush task determined {} are waiting to be flushed.  It has been {} since the last write to ES", batchItemsSent, time);
+                    flushInternal();
+                }
+            }
+        }, 0, maxMsBeforeFlush * 2, TimeUnit.MILLISECONDS);
         manager = new ElasticsearchClientManager(config);
         client = manager.getClient();
 
@@ -262,62 +284,63 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     }
 
     public void flushInternal() {
-        synchronized (this) {
-            // we do not have a working bulk request, we can just exit here.
-            if (this.bulkRequest == null || batchItemsSent == 0)
-                return;
+        //This is unlocked in the callback
+        lock.writeLock().lock();
+        // we do not have a working bulk request, we can just exit here.
+        if (this.bulkRequest == null || batchItemsSent == 0)
+            return;
 
-            // call the flush command.
-            flush(this.bulkRequest, batchItemsSent, batchSizeInBytes);
+        // call the flush command.
+        flush(this.bulkRequest, batchItemsSent, batchSizeInBytes);
 
-            // null the flush request, this will be created in the 'add' function below
-            this.bulkRequest = null;
+        // null the flush request, this will be created in the 'add' function below
+        this.bulkRequest = null;
 
-            // record the proper statistics, and add it to our totals.
-            this.totalSizeInBytes += this.batchSizeInBytes;
-            this.totalSent += batchItemsSent;
+        // record the proper statistics, and add it to our totals.
+        this.totalSizeInBytes += this.batchSizeInBytes;
+        this.totalSent += batchItemsSent;
 
-            // reset the current batch statistics
-            this.batchSizeInBytes = 0;
-            this.batchItemsSent = 0;
+        // reset the current batch statistics
+        this.batchSizeInBytes = 0;
+        this.batchItemsSent = 0;
 
-            try {
-                int count = 0;
-                if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
-                    /****************************************************************************
-                     * Author:
-                     * Smashew
-                     *
-                     * Date:
-                     * 2013-10-20
-                     *
-                     * Note:
-                     * With the information that we have on hand. We need to develop a heuristic
-                     * that will determine when the cluster is having a problem indexing records
-                     * by telling it to pause and wait for it to catch back up. A
-                     *
-                     * There is an impact to us, the caller, whenever this happens as well. Items
-                     * that are not yet fully indexed by the server sit in a queue, on the client
-                     * that can cause the heap to overflow. This has been seen when re-indexing
-                     * large amounts of data to a small cluster. The "deletes" + "indexes" can
-                     * cause the server to have many 'outstandingItems" in queue. Running this
-                     * software with large amounts of data, on a small cluster, can re-create
-                     * this problem.
-                     *
-                     * DO NOT DELETE THESE LINES
-                     ****************************************************************************/
-
-                    // wait for the flush to catch up. We are going to cap this at
-                    while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
-                        Thread.sleep(10);
-
-                    if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
-                        LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
-                }
-            } catch (Exception e) {
-                LOGGER.info("We were broken from our loop: {}", e.getMessage());
+        try {
+            int count = 0;
+            if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
+                /****************************************************************************
+                 * Author:
+                 * Smashew
+                 *
+                 * Date:
+                 * 2013-10-20
+                 *
+                 * Note:
+                 * With the information that we have on hand. We need to develop a heuristic
+                 * that will determine when the cluster is having a problem indexing records
+                 * by telling it to pause and wait for it to catch back up. A
+                 *
+                 * There is an impact to us, the caller, whenever this happens as well. Items
+                 * that are not yet fully indexed by the server sit in a queue, on the client
+                 * that can cause the heap to overflow. This has been seen when re-indexing
+                 * large amounts of data to a small cluster. The "deletes" + "indexes" can
+                 * cause the server to have many 'outstandingItems" in queue. Running this
+                 * software with large amounts of data, on a small cluster, can re-create
+                 * this problem.
+                 *
+                 * DO NOT DELETE THESE LINES
+                 ****************************************************************************/
+
+                // wait for the flush to catch up. We are going to cap this at
+                while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
+                    Thread.sleep(10);
+
+                if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
+                    LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
             }
+        } catch (Exception e) {
+            LOGGER.info("We were broken from our loop: {}", e.getMessage());
         }
+
     }
 
     public void add(String indexName, String type, String json) {
@@ -353,32 +376,35 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     public void add(UpdateRequest updateRequest) {
         Preconditions.checkNotNull(updateRequest);
-        synchronized (this) {
-            checkAndCreateBulkRequest();
-            checkIndexImplications(updateRequest.index());
-            bulkRequest.add(updateRequest);
-            try {
-                Optional<Integer> size = Objects.firstNonNull(
-                        Optional.fromNullable(updateRequest.doc().source().length()),
-                        Optional.fromNullable(updateRequest.script().length()));
-                trackItemAndBytesWritten(size.get().longValue());
-            } catch (NullPointerException x) {
-                trackItemAndBytesWritten(1000);
-            }
+        lock.writeLock().lock();
+        checkAndCreateBulkRequest();
+        checkIndexImplications(updateRequest.index());
+        bulkRequest.add(updateRequest);
+        try {
+            Optional<Integer> size = Objects.firstNonNull(
+                    Optional.fromNullable(updateRequest.doc().source().length()),
+                    Optional.fromNullable(updateRequest.script().length()));
+            trackItemAndBytesWritten(size.get().longValue());
+        } catch (NullPointerException x) {
+            trackItemAndBytesWritten(1000);
+        } finally {
+            lock.writeLock().unlock();
         }
     }
 
     public void add(IndexRequest indexRequest) {
-        synchronized (this) {
-            checkAndCreateBulkRequest();
-            checkIndexImplications(indexRequest.index());
-            bulkRequest.add(indexRequest);
-            try {
-                trackItemAndBytesWritten(indexRequest.source().length());
-            } catch (NullPointerException x) {
-                LOGGER.warn("NPE adding/sizing indexrequest");
-            }
+        lock.writeLock().lock();
+        checkAndCreateBulkRequest();
+        checkIndexImplications(indexRequest.index());
+        bulkRequest.add(indexRequest);
+        try {
+            trackItemAndBytesWritten(indexRequest.source().length());
+        } catch (NullPointerException x) {
+            LOGGER.warn("NPE adding/sizing indexrequest");
+        } finally {
+            lock.writeLock().unlock();
         }
+
     }
 
     public void createIndexIfMissing(String indexName) {
@@ -447,6 +473,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     }
 
     private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+        LOGGER.debug("Attempting to write {} items to ES", thisSent);
         bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
             @Override
             public void onResponse(BulkResponse bulkItemResponses) {
@@ -473,6 +500,9 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
                 if (thisSent != (thisOk + thisFailed))
                     LOGGER.error("We sent more items than this");
 
+                lastWrite.set(System.currentTimeMillis());
+                lock.writeLock().unlock();
+
                 LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
                         MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
                         MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
@@ -480,12 +510,11 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
             @Override
             public void onFailure(Throwable e) {
+                lock.writeLock().unlock();
                 LOGGER.error("Error bulk loading: {}", e.getMessage());
                 e.printStackTrace();
             }
         });
-
-        this.notify();
     }
 
     private void trackItemAndBytesWritten(long sizeInBytes) {
@@ -499,9 +528,12 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
     private void checkAndCreateBulkRequest() {
         // Synchronize to ensure that we don't lose any records
-        synchronized (this) {
+        lock.writeLock().lock();
+        try {
             if (bulkRequest == null)
                 bulkRequest = this.manager.getClient().prepareBulk();
+        } finally {
+            lock.writeLock().unlock();
         }
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/c837a2cf/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
index 21aad5c..c56aa82 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/jsonschema/org/apache/streams/elasticsearch/ElasticsearchWriterConfiguration.json
@@ -13,6 +13,10 @@
         "type": {
             "type": "string",
             "description": "Type to write as"
-        }
+        },
+		"MaxTimeBetweenFlushMs": {
+			"type": "String",
+			"format": "utc-millisec"
+		}
     }
 }
\ No newline at end of file


[12/14] git commit: increasing queue size and initializing twitter during prepare

Posted by sb...@apache.org.
increasing queue size and initializing twitter during prepare


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/64cd1227
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/64cd1227
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/64cd1227

Branch: refs/heads/master
Commit: 64cd1227caba30021f1cfc0d19e5521ce4c80cf4
Parents: 87bd8a3
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 19:27:39 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 19:27:39 2014 -0500

----------------------------------------------------------------------
 .../apache/streams/twitter/provider/TwitterTimelineProvider.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/64cd1227/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index 3799737..e4b0460 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -54,7 +54,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
         this.config = config;
     }
 
-    protected final Queue<StreamsDatum> providerQueue = Queues.synchronizedQueue(new ArrayBlockingQueue<StreamsDatum>(500));
+    protected final Queue<StreamsDatum> providerQueue = Queues.synchronizedQueue(new ArrayBlockingQueue<StreamsDatum>(5000));
 
     protected int idsCount;
     protected Twitter client;
@@ -245,6 +245,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
         jsonStoreEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getJsonStoreEnabled()))).or(true);
         includeEntitiesEnabled = Optional.fromNullable(new Boolean(Boolean.parseBoolean(config.getIncludeEntities()))).or(true);
 
+        client = getTwitterClient();
     }
 
     


[09/14] git commit: was missing imports and using random in a way that wouldn't build in jdk6

Posted by sb...@apache.org.
was missing imports and using random in a way that wouldn't build in jdk6


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/1cc9d0bb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/1cc9d0bb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/1cc9d0bb

Branch: refs/heads/master
Commit: 1cc9d0bb8b54f24f20fa979ba84e9269603b896e
Parents: 145ec84
Author: sblackmon <sb...@w2odigital.com>
Authored: Thu May 8 14:32:14 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Thu May 8 14:32:14 2014 -0500

----------------------------------------------------------------------
 .../streams/twitter/provider/TwitterTimelineProvider.java       | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/1cc9d0bb/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
index db1ec76..799abe7 100644
--- a/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
+++ b/streams-contrib/streams-provider-twitter/src/main/java/org/apache/streams/twitter/provider/TwitterTimelineProvider.java
@@ -23,8 +23,10 @@ import twitter4j.json.DataObjectFactory;
 
 import java.io.Serializable;
 import java.math.BigInteger;
+import java.lang.Math;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Queue;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
@@ -154,7 +156,7 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
         try {
             // wait one tenth of a millisecond
             Thread.yield();
-            Thread.sleep(new Random().nextInt(2));
+            Thread.sleep(1);
             Thread.yield();
         }
         catch(IllegalArgumentException e) {
@@ -260,7 +262,6 @@ public class TwitterTimelineProvider implements StreamsProvider, Serializable {
                 .setAsyncNumThreads(3)
                 .setRestBaseURL(baseUrl)
                 .setIncludeMyRetweetEnabled(Boolean.TRUE)
-                .setIncludeRTsEnabled(Boolean.TRUE)
                 .setPrettyDebugEnabled(Boolean.TRUE);
 
         return new TwitterFactory(builder.build()).getInstance();


[13/14] git commit: line missing from merge

Posted by sb...@apache.org.
line missing from merge


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/237f2a6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/237f2a6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/237f2a6a

Branch: refs/heads/master
Commit: 237f2a6ab13c68a61cf715004fe7b85db66237f6
Parents: 64cd122
Author: sblackmon <sb...@w2odigital.com>
Authored: Fri May 9 15:44:42 2014 -0500
Committer: sblackmon <sb...@w2odigital.com>
Committed: Fri May 9 15:44:42 2014 -0500

----------------------------------------------------------------------
 .../apache/streams/elasticsearch/ElasticsearchPersistWriter.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/237f2a6a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index 97796e2..271521c 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -418,8 +418,10 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
         // If our queue is larger than our flush threashold, then we should flush the queue.
         if( (batchSizeInBytes > flushThresholdSizeInBytes) ||
-                (currentItems >= batchSize) )
+                (currentItems >= batchSize) ) {
             flushInternal();
+            this.currentItems = 0;
+        }
     }
 
     private void checkIndexImplications(String indexName)