You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/04/22 19:13:03 UTC

[1/3] git commit: Updated formatting and organization to better reflect best practices

Repository: incubator-streams
Updated Branches:
  refs/heads/STREAMS-58 [created] f00856946


Updated formatting and organization to better reflect best practices


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/af1efe8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/af1efe8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/af1efe8d

Branch: refs/heads/STREAMS-58
Commit: af1efe8df9a4d91359066757d229cceb19164369
Parents: 3d96497
Author: mfranklin <mf...@apache.org>
Authored: Tue Apr 22 10:04:56 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Apr 22 10:04:56 2014 -0400

----------------------------------------------------------------------
 .../ElasticsearchClientManager.java             | 152 +++----
 .../ElasticsearchConfigurator.java              |   4 +-
 .../ElasticsearchPersistReader.java             | 282 ++++++------
 .../ElasticsearchPersistUpdater.java            | 209 ++++-----
 .../ElasticsearchPersistWriter.java             | 440 +++++++++----------
 .../elasticsearch/PercolateProcessor.java       |  11 +-
 6 files changed, 558 insertions(+), 540 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
index f1d6aad..2f8658a 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
@@ -25,55 +25,110 @@ import java.util.concurrent.ExecutionException;
 /**
  * Created by sblackmon on 2/10/14.
  */
-public class ElasticsearchClientManager
-{
+public class ElasticsearchClientManager {
     private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchClientManager.class);
     private static Map<String, ElasticsearchClient> ALL_CLIENTS = new HashMap<String, ElasticsearchClient>();
 
     private ElasticsearchConfiguration elasticsearchConfiguration;
 
-    public ElasticsearchConfiguration getElasticsearchConfiguration()   { return elasticsearchConfiguration; }
-    public String toString()                                            { return ToStringBuilder.reflectionToString(this); }
-    public boolean equals(Object o)                                     { return EqualsBuilder.reflectionEquals(this, o, Arrays.asList(this.elasticsearchConfiguration.toString())); }
-    public int hashCode()                                               { return HashCodeBuilder.reflectionHashCode(this, Arrays.asList(this.elasticsearchConfiguration.toString())); }
-
     public ElasticsearchClientManager(ElasticsearchConfiguration elasticsearchConfiguration) {
         this.elasticsearchConfiguration = elasticsearchConfiguration;
     }
 
-    /**************************************************************************************
+    public ElasticsearchConfiguration getElasticsearchConfiguration() {
+        return elasticsearchConfiguration;
+    }
+
+    /**
+     * ***********************************************************************************
      * Get the Client for this return, it is actually a transport client, but it is much
      * easier to work with the generic object as this interface likely won't change from
      * elasticsearch. This method is synchronized to block threads from creating
      * too many of these at any given time.
-     * @return
-     * Client for elasticsearch
-     *************************************************************************************/
-    public Client getClient()
-    {
+     *
+     * @return Client for elasticsearch
+     * ***********************************************************************************
+     */
+    public Client getClient() {
         checkAndLoadClient(null);
 
         return ALL_CLIENTS.get(this.elasticsearchConfiguration.getClusterName()).getClient();
     }
 
-    public Client getClient(String clusterName)
-    {
+    public Client getClient(String clusterName) {
         checkAndLoadClient(clusterName);
 
         return ALL_CLIENTS.get(this.elasticsearchConfiguration.getClusterName()).getClient();
     }
 
+    public boolean isOnOrAfterVersion(Version version) {
+        return ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getVersion().onOrAfter(version);
+    }
+
+    public void start() throws Exception {
+        /***********************************************************************
+         * Note:
+         * Everything in these classes is being switched to lazy loading. Within
+         * Heroku you only have 60 seconds to connect, and bind to the service,
+         * and you are only allowed to run in 1Gb of memory. Switching all
+         * of this to lazy loading is how we are fixing some of the issues
+         * if you are having issues with these classes, please, refactor
+         * and create a UNIT TEST CASE!!!!!! To ensure that everything is
+         * working before you check it back in.
+         *
+         * Author: Smashew @ 2013-08-26
+         **********************************************************************/
+    }
+
+    public boolean refresh(String index) {
+        return refresh(new String[]{index});
+    }
+
+    public boolean refresh(String[] indexes) {
+        RefreshResponse refreshResponse = this.getClient().admin().indices().prepareRefresh(indexes).execute().actionGet();
+        return refreshResponse.getFailedShards() == 0;
+    }
+
+    public synchronized void stop() {
+        // Terminate the elasticsearch cluster
+        // Check to see if we have a client.
+        if (ALL_CLIENTS.containsKey(this.elasticsearchConfiguration.toString())) {
+            // Close the client
+            ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getClient().close();
+
+            // Remove it so that it isn't in memory any more.
+            ALL_CLIENTS.remove(this.elasticsearchConfiguration.toString());
+        }
+    }
+
+    public ClusterHealthResponse getStatus() throws ExecutionException, InterruptedException {
+        return new ClusterHealthRequestBuilder(this.getClient().admin().cluster())
+                .execute()
+                .get();
+    }
+
+    public String toString() {
+        return ToStringBuilder.reflectionToString(this);
+    }
+
+    public boolean equals(Object o) {
+        return EqualsBuilder.reflectionEquals(this, o, Arrays.asList(this.elasticsearchConfiguration.toString()));
+    }
+
+    public int hashCode() {
+        return HashCodeBuilder.reflectionHashCode(this, Arrays.asList(this.elasticsearchConfiguration.toString()));
+    }
+
     private synchronized void checkAndLoadClient(String clusterName) {
 
-        if( clusterName == null )
+        if (clusterName == null)
             clusterName = this.elasticsearchConfiguration.getClusterName();
 
         // If it is there, exit early
         if (ALL_CLIENTS.containsKey(clusterName))
             return;
 
-        try
-        {
+        try {
             // We are currently using lazy loading to start the elasticsearch cluster, however.
             LOGGER.info("Creating a new TransportClient: {}", this.elasticsearchConfiguration.getHosts());
 
@@ -86,7 +141,7 @@ public class ElasticsearchClientManager
 
             // Create the client
             TransportClient client = new TransportClient(settings);
-            for(String h : this.getElasticsearchConfiguration().getHosts()) {
+            for (String h : this.getElasticsearchConfiguration().getHosts()) {
                 LOGGER.info("Adding Host: {}", h);
                 client.addTransportAddress(new InetSocketTransportAddress(h, this.getElasticsearchConfiguration().getPort().intValue()));
             }
@@ -97,75 +152,20 @@ public class ElasticsearchClientManager
             // Add it to our static map
             ALL_CLIENTS.put(clusterName, elasticsearchClient);
 
-        }
-        catch(Exception e)
-        {
+        } catch (Exception e) {
             LOGGER.error("Could not Create elasticsearch Transport Client: {}", e);
         }
 
     }
 
-
     private Version getVersion(Client client) {
         try {
             ClusterStateRequestBuilder clusterStateRequestBuilder = new ClusterStateRequestBuilder(client.admin().cluster());
             ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
 
             return clusterStateResponse.getState().getNodes().getMasterNode().getVersion();
-        }
-        catch (Exception e) {
+        } catch (Exception e) {
             return null;
         }
     }
-
-
-    public boolean isOnOrAfterVersion(Version version) {
-        return ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getVersion().onOrAfter(version);
-    }
-
-    public void start() throws Exception
-    {
-        /***********************************************************************
-         * Note:
-         * Everything in these classes is being switched to lazy loading. Within
-         * Heroku you only have 60 seconds to connect, and bind to the service,
-         * and you are only allowed to run in 1Gb of memory. Switching all
-         * of this to lazy loading is how we are fixing some of the issues
-         * if you are having issues with these classes, please, refactor
-         * and create a UNIT TEST CASE!!!!!! To ensure that everything is
-         * working before you check it back in.
-         *
-         * Author: Smashew @ 2013-08-26
-         **********************************************************************/
-    }
-
-    public boolean refresh(String index) {
-        return refresh(new String[]{index});
-    }
-
-    public boolean refresh(String[] indexes) {
-        RefreshResponse refreshResponse = this.getClient().admin().indices().prepareRefresh(indexes).execute().actionGet();
-        return refreshResponse.getFailedShards() == 0;
-    }
-
-    public synchronized void stop()
-    {
-        // Terminate the elasticsearch cluster
-        // Check to see if we have a client.
-        if (ALL_CLIENTS.containsKey(this.elasticsearchConfiguration.toString()))
-        {
-            // Close the client
-            ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getClient().close();
-
-            // Remove it so that it isn't in memory any more.
-            ALL_CLIENTS.remove(this.elasticsearchConfiguration.toString());
-        }
-    }
-
-    public ClusterHealthResponse getStatus() throws ExecutionException, InterruptedException
-    {
-        return new ClusterHealthRequestBuilder(this.getClient().admin().cluster())
-                .execute()
-                .get();
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 20b5c08..483a177 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@ -33,7 +33,7 @@ public class ElasticsearchConfigurator {
     public static ElasticsearchReaderConfiguration detectReaderConfiguration(Config elasticsearch) {
 
         ElasticsearchConfiguration elasticsearchConfiguration = detectConfiguration(elasticsearch);
-        ElasticsearchReaderConfiguration elasticsearchReaderConfiguration  = mapper.convertValue(elasticsearchConfiguration, ElasticsearchReaderConfiguration.class);
+        ElasticsearchReaderConfiguration elasticsearchReaderConfiguration = mapper.convertValue(elasticsearchConfiguration, ElasticsearchReaderConfiguration.class);
 
         List<String> indexes = elasticsearch.getStringList("indexes");
         List<String> types = elasticsearch.getStringList("types");
@@ -47,7 +47,7 @@ public class ElasticsearchConfigurator {
     public static ElasticsearchWriterConfiguration detectWriterConfiguration(Config elasticsearch) {
 
         ElasticsearchConfiguration elasticsearchConfiguration = detectConfiguration(elasticsearch);
-        ElasticsearchWriterConfiguration elasticsearchWriterConfiguration  = mapper.convertValue(elasticsearchConfiguration, ElasticsearchWriterConfiguration.class);
+        ElasticsearchWriterConfiguration elasticsearchWriterConfiguration = mapper.convertValue(elasticsearchConfiguration, ElasticsearchWriterConfiguration.class);
 
         String index = elasticsearch.getString("index");
         String type = elasticsearch.getString("type");

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
index ea74bf6..72a9954 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
@@ -1,14 +1,9 @@
 package org.apache.streams.elasticsearch;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Objects;
-import com.google.common.collect.Iterators;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Queues;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
 import com.typesafe.config.Config;
 import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.core.*;
@@ -25,29 +20,28 @@ import org.joda.time.DateTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.IOException;
 import java.math.BigInteger;
 import java.util.*;
 import java.util.concurrent.*;
 
-/**************************************************************************************************************
+/**
+ * ***********************************************************************************************************
  * Authors:
  * smashew
  * steveblackmon
- **************************************************************************************************************/
+ * ************************************************************************************************************
+ */
 
-public class ElasticsearchPersistReader implements StreamsPersistReader, Iterable<SearchHit>, Iterator<SearchHit>
-{
-    public final static String STREAMS_ID = "ElasticsearchPersistReader";
-
-    private final static Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
-
-    protected volatile Queue<StreamsDatum> persistQueue;
+public class ElasticsearchPersistReader implements StreamsPersistReader, Iterable<SearchHit>, Iterator<SearchHit> {
+    public static final String STREAMS_ID = "ElasticsearchPersistReader";
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
     private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
     private static final Integer DEFAULT_BATCH_SIZE = 500;
     private static final String DEFAULT_SCROLL_TIMEOUT = "5m";
 
+    protected volatile Queue<StreamsDatum> persistQueue;
+
     private ElasticsearchClientManager elasticsearchClientManager;
     private List<String> indexes = Lists.newArrayList();
     private List<String> types = Lists.newArrayList();
@@ -55,7 +49,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
     private String[] withoutfields;
     private DateTime startDate;
     private DateTime endDate;
-    private int limit = 1000*1000*1000; // we are going to set the default limit very high to 1bil
+    private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
     private boolean random = false;
     private int threadPoolSize = 10;
     private int batchSize = 100;
@@ -78,14 +72,32 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
     private long totalHits = 0;
     private long totalRead = 0;
 
-    public long getHitCount()                       { return this.search == null ? 0 : this.totalHits; }
-    public long getReadCount()                      { return this.totalRead; }
-    public double getReadPercent()                  { return (double)this.getReadCount() / (double)this.getHitCount(); }
-    public long getRemainingCount()                 { return this.totalRead - this.totalHits; }
-    public Iterator<SearchHit> iterator()           { return this; }
-    private boolean isCompleted()                   { return totalRead >= this.limit && hasRecords(); }
-    private boolean hasRecords()                    { return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit)); }
-    public SearchHit next()                         { return this.next; }
+    public ElasticsearchPersistReader() {
+        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+        this.config = ElasticsearchConfigurator.detectConfiguration(config);
+    }
+
+    public ElasticsearchPersistReader(ElasticsearchReaderConfiguration elasticsearchConfiguration) {
+        this.elasticsearchClientManager = new ElasticsearchClientManager(elasticsearchConfiguration);
+        indexes.addAll(elasticsearchConfiguration.getIndexes());
+        types.addAll(elasticsearchConfiguration.getTypes());
+    }
+
+    public long getHitCount() {
+        return this.search == null ? 0 : this.totalHits;
+    }
+
+    public long getReadCount() {
+        return this.totalRead;
+    }
+
+    public double getReadPercent() {
+        return (double) this.getReadCount() / (double) this.getHitCount();
+    }
+
+    public long getRemainingCount() {
+        return this.totalRead - this.totalHits;
+    }
 
     public void setBatchSize(int batchSize) {
         this.batchSize = batchSize;
@@ -95,19 +107,23 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         this.scrollTimeout = scrollTimeout;
     }
 
-    public void setQueryBuilder(QueryBuilder queryBuilder)      { this.queryBuilder = queryBuilder; }
-    public void setFilterBuilder(FilterBuilder filterBuilder)   { this.filterBuilder = filterBuilder; }
+    public void setQueryBuilder(QueryBuilder queryBuilder) {
+        this.queryBuilder = queryBuilder;
+    }
 
-    public ElasticsearchPersistReader() {
-        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
-        this.config = ElasticsearchConfigurator.detectConfiguration(config);
+    public void setFilterBuilder(FilterBuilder filterBuilder) {
+        this.filterBuilder = filterBuilder;
     }
-    public ElasticsearchPersistReader(ElasticsearchReaderConfiguration elasticsearchConfiguration) {
-        this.elasticsearchClientManager = new ElasticsearchClientManager(elasticsearchConfiguration);
-        indexes.addAll(elasticsearchConfiguration.getIndexes());
-        types.addAll(elasticsearchConfiguration.getTypes());
+
+    public void setWithfields(String[] withfields) {
+        this.withfields = withfields;
+    }
+
+    public void setWithoutfields(String[] withoutfields) {
+        this.withoutfields = withoutfields;
     }
 
+    //PersistReader methods
     @Override
     public void startStream() {
         LOGGER.debug("startStream");
@@ -122,26 +138,25 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
 
         // If we haven't already set up the search, then set up the search.
-        if(search == null)
-        {
+        if (search == null) {
             search = elasticsearchClientManager.getClient()
                     .prepareSearch(indexes.toArray(new String[0]))
                     .setSearchType(SearchType.SCAN)
                     .setSize(Objects.firstNonNull(batchSize, DEFAULT_BATCH_SIZE).intValue())
                     .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT));
 
-            if(this.queryBuilder != null)
+            if (this.queryBuilder != null)
                 search.setQuery(this.queryBuilder);
 
             // If the types are null, then don't specify a type
-            if(this.types != null && this.types.size() > 0)
+            if (this.types != null && this.types.size() > 0)
                 search = search.setTypes(types.toArray(new String[0]));
 
             Integer clauses = 0;
-            if(this.withfields != null || this.withoutfields != null) {
-                if( this.withfields != null )
+            if (this.withfields != null || this.withoutfields != null) {
+                if (this.withfields != null)
                     clauses += this.withfields.length;
-                if( this.withoutfields != null )
+                if (this.withoutfields != null)
                     clauses += this.withoutfields.length;
             }
 
@@ -149,44 +164,86 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
 
             FilterBuilder allFilters = andFilters(filterList);
 
-            if( clauses > 0 ) {
+            if (clauses > 0) {
                 //    search.setPostFilter(allFilters);
                 search.setPostFilter(allFilters);
             }
 
             // TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
-            if(this.random)
+            if (this.random)
                 search = search.addSort(SortBuilders.scriptSort("random()", "number"));
         }
 
         // We don't have a scroll, we need to create a scroll
-        if(scrollResp == null) {
+        if (scrollResp == null) {
             scrollResp = search.execute().actionGet();
             LOGGER.trace(search.toString());
         }
     }
 
-    public void setWithfields(String[] withfields) {
-        this.withfields = withfields;
+    @Override
+    public StreamsResultSet readAll() {
+        return readCurrent();
     }
 
-    public void setWithoutfields(String[] withoutfields) {
-        this.withoutfields = withoutfields;
+    @Override
+    public StreamsResultSet readCurrent() {
+
+        StreamsResultSet current;
+
+        synchronized (ElasticsearchPersistReader.class) {
+            current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(persistQueue));
+            current.setCounter(new DatumStatusCounter());
+//            current.getCounter().add(countersCurrent);
+//            countersTotal.add(countersCurrent);
+//            countersCurrent = new DatumStatusCounter();
+            persistQueue.clear();
+        }
+
+        return current;
+
+    }
+
+    //TODO - This just reads current records and does not adjust any queries
+    @Override
+    public StreamsResultSet readNew(BigInteger sequence) {
+        return readCurrent();
+    }
+
+    //TODO - This just reads current records and does not adjust any queries
+    @Override
+    public StreamsResultSet readRange(DateTime start, DateTime end) {
+        return readCurrent();
+    }
+
+    @Override
+    public void cleanUp() {
+        LOGGER.info("PersistReader done");
+    }
+
+    //Iterable methods
+    @Override
+    public Iterator<SearchHit> iterator() {
+        return this;
+    }
+
+    //Iterator methods
+    @Override
+    public SearchHit next() {
+        return this.next;
     }
 
-    public boolean hasNext()
-    {
+    @Override
+    public boolean hasNext() {
         calcNext();
         return hasRecords();
     }
 
-    public void calcNext()
-    {
-        try
-        {
+
+    public void calcNext() {
+        try {
             // We have exhausted our scroll create another scroll.
-            if(scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length)
-            {
+            if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
                 // reset the scroll position
                 scrollPositionInScroll = 0;
 
@@ -202,10 +259,9 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
 
             // If this scroll has 0 items then we set the scroll position to -1
             // letting the iterator know that we are done.
-            if(scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
+            if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
                 scrollPositionInScroll = -1;
-            else
-            {
+            else {
                 // get the next record
                 next = scrollResp.getHits().getAt(scrollPositionInScroll);
 
@@ -213,9 +269,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
                 scrollPositionInScroll += 1;
                 totalRead += 1;
             }
-        }
-        catch(Exception e)
-        {
+        } catch (Exception e) {
             e.printStackTrace();
             LOGGER.error("Unexpected scrolling error: {}", e.getMessage());
             scrollPositionInScroll = -1;
@@ -223,31 +277,56 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         }
     }
 
-    public void remove() { }
+    public void remove() {
+    }
+
+    void shutdownAndAwaitTermination(ExecutorService pool) {
+        pool.shutdown(); // Disable new tasks from being submitted
+        try {
+            // Wait a while for existing tasks to terminate
+            if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
+                pool.shutdownNow(); // Cancel currently executing tasks
+                // Wait a while for tasks to respond to being cancelled
+                if (!pool.awaitTermination(10, TimeUnit.SECONDS))
+                    System.err.println("Pool did not terminate");
+            }
+        } catch (InterruptedException ie) {
+            // (Re-)Cancel if current thread also interrupted
+            pool.shutdownNow();
+            // Preserve interrupt status
+            Thread.currentThread().interrupt();
+        }
+    }
+
+    private boolean isCompleted() {
+        return totalRead >= this.limit && hasRecords();
+    }
+
+    private boolean hasRecords() {
+        return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
+    }
 
     // copied from elasticsearch
     // if we need this again we should factor it out into a utility
-    private FilterBuilder andFilters(List<FilterBuilder> filters)
-    {
-        if(filters == null || filters.size() == 0)
+    private FilterBuilder andFilters(List<FilterBuilder> filters) {
+        if (filters == null || filters.size() == 0)
             return null;
 
         FilterBuilder toReturn = filters.get(0);
 
-        for(int i = 1; i < filters.size(); i++)
+        for (int i = 1; i < filters.size(); i++)
             toReturn = FilterBuilders.andFilter(toReturn, filters.get(i));
 
         return toReturn;
     }
 
-    private FilterBuilder orFilters(List<FilterBuilder> filters)
-    {
-        if(filters == null || filters.size() == 0)
+    private FilterBuilder orFilters(List<FilterBuilder> filters) {
+        if (filters == null || filters.size() == 0)
             return null;
 
         FilterBuilder toReturn = filters.get(0);
 
-        for(int i = 1; i < filters.size(); i++)
+        for (int i = 1; i < filters.size(); i++)
             toReturn = FilterBuilders.orFilter(toReturn, filters.get(i));
 
         return toReturn;
@@ -259,9 +338,9 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
 
         // If any withfields are specified, require that field be present
         //    There must a value set also for the document to be processed
-        if(this.withfields != null && this.withfields.length > 0) {
+        if (this.withfields != null && this.withfields.length > 0) {
             ArrayList<FilterBuilder> withFilterList = Lists.newArrayList();
-            for( String withfield : this.withfields ) {
+            for (String withfield : this.withfields) {
                 FilterBuilder withFilter = FilterBuilders.existsFilter(withfield);
                 withFilterList.add(withFilter);
             }
@@ -272,9 +351,9 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         //    Document will be picked up even if present, if they do not have at least one value
         // this is annoying as it majorly impacts runtime
         // might be able to change behavior using null_field
-        if(this.withoutfields != null && this.withoutfields.length > 0) {
+        if (this.withoutfields != null && this.withoutfields.length > 0) {
             ArrayList<FilterBuilder> withoutFilterList = Lists.newArrayList();
-            for( String withoutfield : this.withoutfields ) {
+            for (String withoutfield : this.withoutfields) {
                 FilterBuilder withoutFilter = FilterBuilders.missingFilter(withoutfield).existence(true).nullValue(false);
                 withoutFilterList.add(withoutFilter);
             }
@@ -285,61 +364,6 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         return filterList;
     }
 
-    @Override
-    public StreamsResultSet readCurrent() {
-
-        StreamsResultSet current;
-
-        synchronized( ElasticsearchPersistReader.class ) {
-            current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(persistQueue));
-            current.setCounter(new DatumStatusCounter());
-//            current.getCounter().add(countersCurrent);
-//            countersTotal.add(countersCurrent);
-//            countersCurrent = new DatumStatusCounter();
-            persistQueue.clear();
-        }
-
-        return current;
-
-    }
-
-    public StreamsResultSet readAll() {
-        return readCurrent();
-    }
-
-    @Override
-    public StreamsResultSet readNew(BigInteger sequence) {
-        return readCurrent();
-    }
-
-    @Override
-    public StreamsResultSet readRange(DateTime start, DateTime end) {
-        return readCurrent();
-    }
-
-    void shutdownAndAwaitTermination(ExecutorService pool) {
-        pool.shutdown(); // Disable new tasks from being submitted
-        try {
-            // Wait a while for existing tasks to terminate
-            if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
-                pool.shutdownNow(); // Cancel currently executing tasks
-                // Wait a while for tasks to respond to being cancelled
-                if (!pool.awaitTermination(10, TimeUnit.SECONDS))
-                    System.err.println("Pool did not terminate");
-            }
-        } catch (InterruptedException ie) {
-            // (Re-)Cancel if current thread also interrupted
-            pool.shutdownNow();
-            // Preserve interrupt status
-            Thread.currentThread().interrupt();
-        }
-    }
-
-    @Override
-    public void cleanUp() {
-        LOGGER.info("PersistReader done");
-    }
-
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
index 5f59950..662a084 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
@@ -41,8 +41,7 @@ import java.util.*;
 
 //import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
 
-public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable
-{
+public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable {
     private final static Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistUpdater.class);
     private final static NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
     private final static NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
@@ -92,19 +91,53 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
 
     private final List<String> affectedIndexes = new ArrayList<String>();
 
-    public int getTotalOutstanding()                           { return this.totalSent - (this.totalFailed + this.totalOk); }
-    public long getFlushThresholdSizeInBytes()                 { return flushThresholdSizeInBytes; }
-    public int getTotalSent()                                  { return totalSent; }
-    public int getTotalSeconds()                               { return totalSeconds; }
-    public int getTotalOk()                                    { return totalOk; }
-    public int getTotalFailed()                                { return totalFailed; }
-    public int getTotalBatchCount()                            { return totalBatchCount; }
-    public long getTotalSizeInBytes()                          { return totalSizeInBytes; }
-    public long getBatchSizeInBytes()                          { return batchSizeInBytes; }
-    public int getBatchItemsSent()                             { return batchItemsSent; }
-    public List<String> getAffectedIndexes()                   { return this.affectedIndexes; }
+    public int getTotalOutstanding() {
+        return this.totalSent - (this.totalFailed + this.totalOk);
+    }
+
+    public long getFlushThresholdSizeInBytes() {
+        return flushThresholdSizeInBytes;
+    }
+
+    public int getTotalSent() {
+        return totalSent;
+    }
+
+    public int getTotalSeconds() {
+        return totalSeconds;
+    }
+
+    public int getTotalOk() {
+        return totalOk;
+    }
+
+    public int getTotalFailed() {
+        return totalFailed;
+    }
+
+    public int getTotalBatchCount() {
+        return totalBatchCount;
+    }
+
+    public long getTotalSizeInBytes() {
+        return totalSizeInBytes;
+    }
+
+    public long getBatchSizeInBytes() {
+        return batchSizeInBytes;
+    }
+
+    public int getBatchItemsSent() {
+        return batchItemsSent;
+    }
+
+    public List<String> getAffectedIndexes() {
+        return this.affectedIndexes;
+    }
 
-    public void setFlushThresholdSizeInBytes(long sizeInBytes)  { this.flushThresholdSizeInBytes = sizeInBytes; }
+    public void setFlushThresholdSizeInBytes(long sizeInBytes) {
+        this.flushThresholdSizeInBytes = sizeInBytes;
+    }
 
     Thread task;
 
@@ -123,12 +156,14 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
         this.config = config;
     }
 
-    private static final int  BYTES_IN_MB = 1024*1024;
-    private static final int  BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
-    private volatile int  totalByteCount = 0;
-    private volatile int  byteCount = 0;
+    private static final int BYTES_IN_MB = 1024 * 1024;
+    private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+    private volatile int totalByteCount = 0;
+    private volatile int byteCount = 0;
 
-    public boolean isConnected() 		                { return (client != null); }
+    public boolean isConnected() {
+        return (client != null);
+    }
 
     @Override
     public void write(StreamsDatum streamsDatum) {
@@ -173,30 +208,25 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
     }
 
     @Override
-    public void close()
-    {
-        try
-        {
+    public void close() {
+        try {
             // before they close, check to ensure that
             this.flush();
 
             int count = 0;
             // We are going to give it 5 minutes.
-            while(this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
+            while (this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
                 Thread.sleep(50);
 
-            if(this.getTotalOutstanding() > 0)
-            {
+            if (this.getTotalOutstanding() > 0) {
                 LOGGER.error("We never cleared our buffer");
             }
 
 
-            for(String indexName : this.getAffectedIndexes())
-            {
+            for (String indexName : this.getAffectedIndexes()) {
                 createIndexIfMissing(indexName);
 
-                if(this.veryLargeBulk)
-                {
+                if (this.veryLargeBulk) {
                     LOGGER.debug("Resetting our Refresh Interval: {}", indexName);
                     // They are in 'very large bulk' mode and the process is finished. We now want to turn the
                     // refreshing back on.
@@ -224,9 +254,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
 
             LOGGER.info("Closed: Wrote[{} of {}] Failed[{}]", this.getTotalOk(), this.getTotalSent(), this.getTotalFailed());
 
-        }
-        catch(Exception e)
-        {
+        } catch (Exception e) {
             // this line of code should be logically unreachable.
             LOGGER.warn("This is unexpected: {}", e.getMessage());
             e.printStackTrace();
@@ -234,17 +262,14 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
     }
 
     @Override
-    public void flush() throws IOException
-    {
+    public void flush() throws IOException {
         flushInternal();
     }
 
-    public void flushInternal()
-    {
-        synchronized (this)
-        {
+    public void flushInternal() {
+        synchronized (this) {
             // we do not have a working bulk request, we can just exit here.
-            if(this.bulkRequest == null || batchItemsSent == 0)
+            if (this.bulkRequest == null || batchItemsSent == 0)
                 return;
 
             // call the flush command.
@@ -261,11 +286,9 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
             this.batchSizeInBytes = 0;
             this.batchItemsSent = 0;
 
-            try
-            {
+            try {
                 int count = 0;
-                if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
-                {
+                if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
                     /****************************************************************************
                      * Author:
                      * Smashew
@@ -290,27 +313,22 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
                      ****************************************************************************/
 
                     // wait for the flush to catch up. We are going to cap this at
-                    while(this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
+                    while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
                         Thread.sleep(10);
 
-                    if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
+                    if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
                         LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
                 }
-            }
-            catch(Exception e)
-            {
+            } catch (Exception e) {
                 LOGGER.info("We were broken from our loop: {}", e.getMessage());
             }
         }
     }
 
-    private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes)
-    {
-        bulkRequest.execute().addListener(new ActionListener<BulkResponse>()
-        {
+    private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+        bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
             @Override
-            public void onResponse(BulkResponse bulkItemResponses)
-            {
+            public void onResponse(BulkResponse bulkItemResponses) {
                 if (bulkItemResponses.hasFailures())
                     LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
 
@@ -319,9 +337,8 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
                 long thisMillis = bulkItemResponses.getTookInMillis();
 
                 // keep track of the number of totalFailed and items that we have totalOk.
-                for(BulkItemResponse resp : bulkItemResponses.getItems())
-                {
-                    if(resp.isFailed())
+                for (BulkItemResponse resp : bulkItemResponses.getItems()) {
+                    if (resp.isFailed())
                         thisFailed++;
                     else
                         thisOk++;
@@ -331,17 +348,16 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
                 totalFailed += thisFailed;
                 totalSeconds += (thisMillis / 1000);
 
-                if(thisSent != (thisOk + thisFailed))
+                if (thisSent != (thisOk + thisFailed))
                     LOGGER.error("We sent more items than this");
 
                 LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
-                        MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
-                        MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+                        MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
+                        MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
             }
 
             @Override
-            public void onFailure(Throwable e)
-            {
+            public void onFailure(Throwable e) {
                 LOGGER.error("Error bulk loading: {}", e.getMessage());
                 e.printStackTrace();
             }
@@ -350,8 +366,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
         this.notify();
     }
 
-    public void add(String indexName, String type, String id, String json)
-    {
+    public void add(String indexName, String type, String id, String json) {
         UpdateRequest updateRequest;
 
         Preconditions.checkNotNull(id);
@@ -359,20 +374,18 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
 
         // They didn't specify an ID, so we will create one for them.
         updateRequest = new UpdateRequest()
-            .index(indexName)
-            .type(type)
-            .id(id)
-            .doc(json);
+                .index(indexName)
+                .type(type)
+                .id(id)
+                .doc(json);
 
         add(updateRequest);
 
     }
 
-    public void add(UpdateRequest updateRequest)
-    {
+    public void add(UpdateRequest updateRequest) {
         Preconditions.checkNotNull(updateRequest);
-        synchronized (this)
-        {
+        synchronized (this) {
             checkAndCreateBulkRequest();
             checkIndexImplications(updateRequest.index());
             bulkRequest.add(updateRequest);
@@ -381,37 +394,33 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
                         Optional.fromNullable(updateRequest.doc().source().length()),
                         Optional.fromNullable(updateRequest.script().length()));
                 trackItemAndBytesWritten(size.get().longValue());
-            } catch( NullPointerException x) {
+            } catch (NullPointerException x) {
                 trackItemAndBytesWritten(1000);
             }
         }
     }
 
-    private void trackItemAndBytesWritten(long sizeInBytes)
-    {
+    private void trackItemAndBytesWritten(long sizeInBytes) {
         batchItemsSent++;
         batchSizeInBytes += sizeInBytes;
 
         // If our queue is larger than our flush threashold, then we should flush the queue.
-        if(batchSizeInBytes > flushThresholdSizeInBytes)
+        if (batchSizeInBytes > flushThresholdSizeInBytes)
             flushInternal();
     }
 
-    private void checkAndCreateBulkRequest()
-    {
+    private void checkAndCreateBulkRequest() {
         // Synchronize to ensure that we don't lose any records
-        synchronized (this)
-        {
-            if(bulkRequest == null)
+        synchronized (this) {
+            if (bulkRequest == null)
                 bulkRequest = this.manager.getClient().prepareBulk();
         }
     }
 
-    private void checkIndexImplications(String indexName)
-    {
+    private void checkIndexImplications(String indexName) {
 
         // check to see if we have seen this index before.
-        if(this.affectedIndexes.contains(indexName))
+        if (this.affectedIndexes.contains(indexName))
             return;
 
         // we haven't log this index.
@@ -419,7 +428,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
 
         // Check to see if we are in 'veryLargeBulk' mode
         // if we aren't, exit early
-        if(!this.veryLargeBulk)
+        if (!this.veryLargeBulk)
             return;
 
 
@@ -438,24 +447,20 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
     }
 
     public void createIndexIfMissing(String indexName) {
-        if(!this.manager.getClient()
+        if (!this.manager.getClient()
                 .admin()
                 .indices()
                 .exists(new IndicesExistsRequest(indexName))
                 .actionGet()
-                .isExists())
-        {
+                .isExists()) {
             // It does not exist... So we are going to need to create the index.
             // we are going to assume that the 'templates' that we have loaded into
             // elasticsearch are sufficient to ensure the index is being created properly.
             CreateIndexResponse response = this.manager.getClient().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
 
-            if(response.isAcknowledged())
-            {
+            if (response.isAcknowledged()) {
                 LOGGER.info("Index {} did not exist. The index was automatically created from the stored ElasticSearch Templates.", indexName);
-            }
-            else
-            {
+            } else {
                 LOGGER.error("Index {} did not exist. While attempting to create the index from stored ElasticSearch Templates we were unable to get an acknowledgement.", indexName);
                 LOGGER.error("Error Message: {}", response.toString());
                 throw new RuntimeException("Unable to create index " + indexName);
@@ -463,18 +468,16 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
         }
     }
 
-    public void add(String indexName, String type, Map<String, Object> toImport)
-    {
+    public void add(String indexName, String type, Map<String, Object> toImport) {
         for (String id : toImport.keySet())
-            add(indexName, type, id, (String)toImport.get(id));
+            add(indexName, type, id, (String) toImport.get(id));
     }
 
-    private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch)
-    {
+    private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch) {
         Set<String> invalidIDs = checkIds(workingBatch.keySet(), index, type);
 
-        for(String toAddId : workingBatch.keySet())
-            if(!invalidIDs.contains(toAddId))
+        for (String toAddId : workingBatch.keySet())
+            if (!invalidIDs.contains(toAddId))
                 add(index, type, toAddId, workingBatch.get(toAddId));
 
         LOGGER.info("Adding Batch: {} -> {}", workingBatch.size(), invalidIDs.size());
@@ -485,7 +488,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
 
         IdsQueryBuilder idsFilterBuilder = new IdsQueryBuilder();
 
-        for(String s : input)
+        for (String s : input)
             idsFilterBuilder.addIds(s);
 
         SearchRequestBuilder searchRequestBuilder = this.manager.getClient()
@@ -501,7 +504,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
 
         Set<String> toReturn = new HashSet<String>();
 
-        for(SearchHit hit : hits) {
+        for (SearchHit hit : hits) {
             toReturn.add(hit.getId());
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ad89013..ff7e0f5 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -37,28 +37,35 @@ import java.text.DecimalFormat;
 import java.text.NumberFormat;
 import java.util.*;
 
-public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable
-{
-    public final static String STREAMS_ID = "ElasticsearchPersistWriter";
+public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable {
+    public static final String STREAMS_ID = "ElasticsearchPersistWriter";
 
-    private final static Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
-    private final static NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
-    private final static NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
+    public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
+    private static final NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
+    private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
+    private static final Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
+    private static final long WAITING_DOCS_LIMIT = 10000;
+    private static final int BYTES_IN_MB = 1024 * 1024;
+    private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+
+    private final List<String> affectedIndexes = new ArrayList<String>();
 
+    private ObjectMapper mapper = new StreamsJacksonMapper();
     private ElasticsearchClientManager manager;
+    private ElasticsearchWriterConfiguration config;
     private Client client;
     private String parentID = null;
     private BulkRequestBuilder bulkRequest;
     private OutputStreamWriter currentWriter = null;
-
     private int batchSize = 50;
     private int totalRecordsWritten = 0;
     private boolean veryLargeBulk = false;  // by default this setting is set to false
 
-    private final static Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
-    private static final long WAITING_DOCS_LIMIT = 10000;
+    protected Thread task;
 
-    public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
+    protected volatile Queue<StreamsDatum> persistQueue;
 
     private volatile int totalSent = 0;
     private volatile int totalSeconds = 0;
@@ -67,9 +74,19 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     private volatile int totalFailed = 0;
     private volatile int totalBatchCount = 0;
     private volatile long totalSizeInBytes = 0;
-
     private volatile long batchSizeInBytes = 0;
     private volatile int batchItemsSent = 0;
+    private volatile int totalByteCount = 0;
+    private volatile int byteCount = 0;
+
+    public ElasticsearchPersistWriter() {
+        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+        this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+    }
+
+    public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
+        this.config = config;
+    }
 
     public void setBatchSize(int batchSize) {
         this.batchSize = batchSize;
@@ -79,45 +96,58 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         this.veryLargeBulk = veryLargeBulk;
     }
 
-    private final List<String> affectedIndexes = new ArrayList<String>();
 
-    public int getTotalOutstanding()                           { return this.totalSent - (this.totalFailed + this.totalOk); }
-    public long getFlushThresholdSizeInBytes()                 { return flushThresholdSizeInBytes; }
-    public int getTotalSent()                                  { return totalSent; }
-    public int getTotalSeconds()                               { return totalSeconds; }
-    public int getTotalOk()                                    { return totalOk; }
-    public int getTotalFailed()                                { return totalFailed; }
-    public int getTotalBatchCount()                            { return totalBatchCount; }
-    public long getTotalSizeInBytes()                          { return totalSizeInBytes; }
-    public long getBatchSizeInBytes()                          { return batchSizeInBytes; }
-    public int getBatchItemsSent()                             { return batchItemsSent; }
-    public List<String> getAffectedIndexes()                   { return this.affectedIndexes; }
+    public int getTotalOutstanding() {
+        return this.totalSent - (this.totalFailed + this.totalOk);
+    }
 
-    public void setFlushThresholdSizeInBytes(long sizeInBytes)  { this.flushThresholdSizeInBytes = sizeInBytes; }
+    public long getFlushThresholdSizeInBytes() {
+        return flushThresholdSizeInBytes;
+    }
 
-    Thread task;
+    public int getTotalSent() {
+        return totalSent;
+    }
 
-    protected volatile Queue<StreamsDatum> persistQueue;
+    public int getTotalSeconds() {
+        return totalSeconds;
+    }
 
-    private ObjectMapper mapper = new StreamsJacksonMapper();
+    public int getTotalOk() {
+        return totalOk;
+    }
 
-    private ElasticsearchWriterConfiguration config;
+    public int getTotalFailed() {
+        return totalFailed;
+    }
 
-    public ElasticsearchPersistWriter() {
-        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
-        this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+    public int getTotalBatchCount() {
+        return totalBatchCount;
     }
 
-    public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
-        this.config = config;
+    public long getTotalSizeInBytes() {
+        return totalSizeInBytes;
     }
 
-    private static final int  BYTES_IN_MB = 1024*1024;
-    private static final int  BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
-    private volatile int  totalByteCount = 0;
-    private volatile int  byteCount = 0;
+    public long getBatchSizeInBytes() {
+        return batchSizeInBytes;
+    }
 
-    public boolean isConnected() 		                { return (client != null); }
+    public int getBatchItemsSent() {
+        return batchItemsSent;
+    }
+
+    public List<String> getAffectedIndexes() {
+        return this.affectedIndexes;
+    }
+
+    public void setFlushThresholdSizeInBytes(long sizeInBytes) {
+        this.flushThresholdSizeInBytes = sizeInBytes;
+    }
+
+    public boolean isConnected() {
+        return (client != null);
+    }
 
     @Override
     public void write(StreamsDatum streamsDatum) {
@@ -125,7 +155,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         String json;
         try {
             String id = streamsDatum.getId();
-            if( streamsDatum.getDocument() instanceof String )
+            if (streamsDatum.getDocument() instanceof String)
                 json = streamsDatum.getDocument().toString();
             else {
                 json = mapper.writeValueAsString(streamsDatum.getDocument());
@@ -139,14 +169,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         }
     }
 
-    public void start() {
-
-        manager = new ElasticsearchClientManager(config);
-        client = manager.getClient();
-
-        LOGGER.info(client.toString());
-    }
-
     public void cleanUp() {
 
         try {
@@ -158,30 +180,25 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     }
 
     @Override
-    public void close()
-    {
-        try
-        {
+    public void close() {
+        try {
             // before they close, check to ensure that
             this.flush();
 
             int count = 0;
             // We are going to give it 5 minutes.
-            while(this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
+            while (this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
                 Thread.sleep(50);
 
-            if(this.getTotalOutstanding() > 0)
-            {
+            if (this.getTotalOutstanding() > 0) {
                 LOGGER.error("We never cleared our buffer");
             }
 
 
-            for(String indexName : this.getAffectedIndexes())
-            {
+            for (String indexName : this.getAffectedIndexes()) {
                 createIndexIfMissing(indexName);
 
-                if(this.veryLargeBulk)
-                {
+                if (this.veryLargeBulk) {
                     LOGGER.debug("Resetting our Refresh Interval: {}", indexName);
                     // They are in 'very large bulk' mode and the process is finished. We now want to turn the
                     // refreshing back on.
@@ -209,9 +226,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
             LOGGER.info("Closed: Wrote[{} of {}] Failed[{}]", this.getTotalOk(), this.getTotalSent(), this.getTotalFailed());
 
-        }
-        catch(Exception e)
-        {
+        } catch (Exception e) {
             // this line of code should be logically unreachable.
             LOGGER.warn("This is unexpected: {}", e.getMessage());
             e.printStackTrace();
@@ -219,17 +234,37 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
     }
 
     @Override
-    public void flush() throws IOException
-    {
+    public void flush() throws IOException {
         flushInternal();
     }
 
-    public void flushInternal()
-    {
-        synchronized (this)
-        {
+    @Override
+    public void prepare(Object configurationObject) {
+        mapper = StreamsJacksonMapper.getInstance();
+        start();
+    }
+
+    @Override
+    public DatumStatusCounter getDatumStatusCounter() {
+        DatumStatusCounter counters = new DatumStatusCounter();
+        counters.incrementAttempt(this.batchItemsSent);
+        counters.incrementStatus(DatumStatus.SUCCESS, this.totalOk);
+        counters.incrementStatus(DatumStatus.FAIL, this.totalFailed);
+        return counters;
+    }
+
+    public void start() {
+
+        manager = new ElasticsearchClientManager(config);
+        client = manager.getClient();
+
+        LOGGER.info(client.toString());
+    }
+
+    public void flushInternal() {
+        synchronized (this) {
             // we do not have a working bulk request, we can just exit here.
-            if(this.bulkRequest == null || batchItemsSent == 0)
+            if (this.bulkRequest == null || batchItemsSent == 0)
                 return;
 
             // call the flush command.
@@ -246,11 +281,9 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
             this.batchSizeInBytes = 0;
             this.batchItemsSent = 0;
 
-            try
-            {
+            try {
                 int count = 0;
-                if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
-                {
+                if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
                     /****************************************************************************
                      * Author:
                      * Smashew
@@ -275,78 +308,27 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
                      ****************************************************************************/
 
                     // wait for the flush to catch up. We are going to cap this at
-                    while(this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
+                    while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
                         Thread.sleep(10);
 
-                    if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
+                    if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
                         LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
                 }
-            }
-            catch(Exception e)
-            {
+            } catch (Exception e) {
                 LOGGER.info("We were broken from our loop: {}", e.getMessage());
             }
         }
     }
 
-    private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes)
-    {
-        bulkRequest.execute().addListener(new ActionListener<BulkResponse>()
-        {
-            @Override
-            public void onResponse(BulkResponse bulkItemResponses)
-            {
-                if (bulkItemResponses.hasFailures())
-                    LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
-
-                long thisFailed = 0;
-                long thisOk = 0;
-                long thisMillis = bulkItemResponses.getTookInMillis();
-
-                // keep track of the number of totalFailed and items that we have totalOk.
-                for(BulkItemResponse resp : bulkItemResponses.getItems())
-                {
-                    if(resp.isFailed())
-                        thisFailed++;
-                    else
-                        thisOk++;
-                }
-
-                totalAttempted += thisSent;
-                totalOk += thisOk;
-                totalFailed += thisFailed;
-                totalSeconds += (thisMillis / 1000);
-
-                if(thisSent != (thisOk + thisFailed))
-                    LOGGER.error("We sent more items than this");
-
-                LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
-                        MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
-                        MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
-            }
-
-            @Override
-            public void onFailure(Throwable e)
-            {
-                LOGGER.error("Error bulk loading: {}", e.getMessage());
-                e.printStackTrace();
-            }
-        });
-
-        this.notify();
-    }
-
-    public void add(String indexName, String type, String json)
-    {
+    public void add(String indexName, String type, String json) {
         add(indexName, type, null, json);
     }
 
-    public void add(String indexName, String type, String id, String json)
-    {
+    public void add(String indexName, String type, String id, String json) {
         IndexRequest indexRequest;
 
         // They didn't specify an ID, so we will create one for them.
-        if(id == null)
+        if (id == null)
             indexRequest = new IndexRequest(indexName, type);
         else
             indexRequest = new IndexRequest(indexName, type, id);
@@ -356,16 +338,12 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         // If there is a parentID that is associated with this bulk, then we are
         // going to have to parse the raw JSON and attempt to dereference
         // what the parent document should be
-        if(parentID != null)
-        {
-            try
-            {
+        if (parentID != null) {
+            try {
                 // The JSONObject constructor can throw an exception, it is called
                 // out explicitly here so we can catch it.
                 indexRequest = indexRequest.parent(new JSONObject(json).getString(parentID));
-            }
-            catch(JSONException e)
-            {
+            } catch (JSONException e) {
                 LOGGER.warn("Malformed JSON, cannot grab parentID: {}@{}[{}]: {}", id, indexName, type, e.getMessage());
                 totalFailed++;
             }
@@ -373,11 +351,9 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
         add(indexRequest);
     }
 
-    public void add(UpdateRequest updateRequest)
-    {
+    public void add(UpdateRequest updateRequest) {
         Preconditions.checkNotNull(updateRequest);
-        synchronized (this)
-        {
+        synchronized (this) {
             checkAndCreateBulkRequest();
             checkIndexImplications(updateRequest.index());
             bulkRequest.add(updateRequest);
@@ -386,114 +362,57 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
                         Optional.fromNullable(updateRequest.doc().source().length()),
                         Optional.fromNullable(updateRequest.script().length()));
                 trackItemAndBytesWritten(size.get().longValue());
-            } catch( NullPointerException x) {
+            } catch (NullPointerException x) {
                 trackItemAndBytesWritten(1000);
             }
         }
     }
 
-    public void add(IndexRequest indexRequest)
-    {
-        synchronized (this)
-        {
+    public void add(IndexRequest indexRequest) {
+        synchronized (this) {
             checkAndCreateBulkRequest();
             checkIndexImplications(indexRequest.index());
             bulkRequest.add(indexRequest);
             try {
                 trackItemAndBytesWritten(indexRequest.source().length());
-            } catch( NullPointerException x) {
+            } catch (NullPointerException x) {
                 LOGGER.warn("NPE adding/sizing indexrequest");
             }
         }
     }
 
-    private void trackItemAndBytesWritten(long sizeInBytes)
-    {
-        batchItemsSent++;
-        batchSizeInBytes += sizeInBytes;
-
-        // If our queue is larger than our flush threashold, then we should flush the queue.
-        if(batchSizeInBytes > flushThresholdSizeInBytes)
-            flushInternal();
-    }
-
-    private void checkAndCreateBulkRequest()
-    {
-        // Synchronize to ensure that we don't lose any records
-        synchronized (this)
-        {
-            if(bulkRequest == null)
-                bulkRequest = this.manager.getClient().prepareBulk();
-        }
-    }
-
-    private void checkIndexImplications(String indexName)
-    {
-
-        // check to see if we have seen this index before.
-        if(this.affectedIndexes.contains(indexName))
-            return;
-
-        // we haven't log this index.
-        this.affectedIndexes.add(indexName);
-
-        // Check to see if we are in 'veryLargeBulk' mode
-        // if we aren't, exit early
-        if(!this.veryLargeBulk)
-            return;
-
-
-        // They are in 'very large bulk' mode we want to turn off refreshing the index.
-
-        // Create a request then add the setting to tell it to stop refreshing the interval
-        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
-        updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
-
-        // submit to ElasticSearch
-        this.manager.getClient()
-                .admin()
-                .indices()
-                .updateSettings(updateSettingsRequest)
-                .actionGet();
-    }
-
     public void createIndexIfMissing(String indexName) {
-        if(!this.manager.getClient()
+        if (!this.manager.getClient()
                 .admin()
                 .indices()
                 .exists(new IndicesExistsRequest(indexName))
                 .actionGet()
-                .isExists())
-        {
+                .isExists()) {
             // It does not exist... So we are going to need to create the index.
             // we are going to assume that the 'templates' that we have loaded into
             // elasticsearch are sufficient to ensure the index is being created properly.
             CreateIndexResponse response = this.manager.getClient().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
 
-            if(response.isAcknowledged())
-            {
+            if (response.isAcknowledged()) {
                 LOGGER.info("Index {} did not exist. The index was automatically created from the stored ElasticSearch Templates.", indexName);
-            }
-            else
-            {
+            } else {
                 LOGGER.error("Index {} did not exist. While attempting to create the index from stored ElasticSearch Templates we were unable to get an acknowledgement.", indexName);
                 LOGGER.error("Error Message: {}", response.toString());
                 throw new RuntimeException("Unable to create index " + indexName);
             }
         }
     }
-    public void add(String indexName, String type, Map<String, Object> toImport)
-    {
+
+    public void add(String indexName, String type, Map<String, Object> toImport) {
         for (String id : toImport.keySet())
-            add(indexName, type, id, (String)toImport.get(id));
+            add(indexName, type, id, (String) toImport.get(id));
     }
 
-    private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch)
-    {
+    private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch) {
         Set<String> invalidIDs = checkIds(workingBatch.keySet(), index, type);
 
-        for(String toAddId : workingBatch.keySet())
-            if(!invalidIDs.contains(toAddId))
+        for (String toAddId : workingBatch.keySet())
+            if (!invalidIDs.contains(toAddId))
                 add(index, type, toAddId, workingBatch.get(toAddId));
 
         LOGGER.info("Adding Batch: {} -> {}", workingBatch.size(), invalidIDs.size());
@@ -504,7 +423,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
         IdsQueryBuilder idsFilterBuilder = new IdsQueryBuilder();
 
-        for(String s : input)
+        for (String s : input)
             idsFilterBuilder.addIds(s);
 
         SearchRequestBuilder searchRequestBuilder = this.manager.getClient()
@@ -520,25 +439,98 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
 
         Set<String> toReturn = new HashSet<String>();
 
-        for(SearchHit hit : hits) {
+        for (SearchHit hit : hits) {
             toReturn.add(hit.getId());
         }
 
         return toReturn;
     }
 
-    @Override
-    public void prepare(Object configurationObject) {
-        mapper = StreamsJacksonMapper.getInstance();
-        start();
+    private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+        bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
+            @Override
+            public void onResponse(BulkResponse bulkItemResponses) {
+                if (bulkItemResponses.hasFailures())
+                    LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
+
+                long thisFailed = 0;
+                long thisOk = 0;
+                long thisMillis = bulkItemResponses.getTookInMillis();
+
+                // keep track of the number of totalFailed and items that we have totalOk.
+                for (BulkItemResponse resp : bulkItemResponses.getItems()) {
+                    if (resp.isFailed())
+                        thisFailed++;
+                    else
+                        thisOk++;
+                }
+
+                totalAttempted += thisSent;
+                totalOk += thisOk;
+                totalFailed += thisFailed;
+                totalSeconds += (thisMillis / 1000);
+
+                if (thisSent != (thisOk + thisFailed))
+                    LOGGER.error("We sent more items than this");
+
+                LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
+                        MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
+                        MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+            }
+
+            @Override
+            public void onFailure(Throwable e) {
+                LOGGER.error("Error bulk loading: {}", e.getMessage());
+                e.printStackTrace();
+            }
+        });
+
+        this.notify();
     }
 
-    @Override
-    public DatumStatusCounter getDatumStatusCounter() {
-        DatumStatusCounter counters = new DatumStatusCounter();
-        counters.incrementAttempt(this.batchItemsSent);
-        counters.incrementStatus(DatumStatus.SUCCESS, this.totalOk);
-        counters.incrementStatus(DatumStatus.FAIL, this.totalFailed);
-        return counters;
+    private void trackItemAndBytesWritten(long sizeInBytes) {
+        batchItemsSent++;
+        batchSizeInBytes += sizeInBytes;
+
+        // If our queue is larger than our flush threashold, then we should flush the queue.
+        if (batchSizeInBytes > flushThresholdSizeInBytes)
+            flushInternal();
+    }
+
+    private void checkAndCreateBulkRequest() {
+        // Synchronize to ensure that we don't lose any records
+        synchronized (this) {
+            if (bulkRequest == null)
+                bulkRequest = this.manager.getClient().prepareBulk();
+        }
+    }
+
+    private void checkIndexImplications(String indexName) {
+
+        // check to see if we have seen this index before.
+        if (this.affectedIndexes.contains(indexName))
+            return;
+
+        // we haven't log this index.
+        this.affectedIndexes.add(indexName);
+
+        // Check to see if we are in 'veryLargeBulk' mode
+        // if we aren't, exit early
+        if (!this.veryLargeBulk)
+            return;
+
+
+        // They are in 'very large bulk' mode we want to turn off refreshing the index.
+
+        // Create a request then add the setting to tell it to stop refreshing the interval
+        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
+        updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
+
+        // submit to ElasticSearch
+        this.manager.getClient()
+                .admin()
+                .indices()
+                .updateSettings(updateSettingsRequest)
+                .actionGet();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
index 72df97c..c10c5d5 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
@@ -30,8 +30,7 @@ import java.util.concurrent.LinkedBlockingQueue;
  * [t.co behavior]      https://dev.twitter.com/docs/tco-redirection-behavior
  */
 
-public class PercolateProcessor implements StreamsProcessor, Runnable
-{
+public class PercolateProcessor implements StreamsProcessor, Runnable {
     private final static Logger LOGGER = LoggerFactory.getLogger(PercolateProcessor.class);
 
     private ObjectMapper mapper = new ObjectMapper();
@@ -85,7 +84,7 @@ public class PercolateProcessor implements StreamsProcessor, Runnable
         String json;
         ObjectNode node;
         // first check for valid json
-        if( entry.getDocument() instanceof String ) {
+        if (entry.getDocument() instanceof String) {
             json = (String) entry.getDocument();
             try {
                 node = (ObjectNode) mapper.readTree(json);
@@ -102,7 +101,7 @@ public class PercolateProcessor implements StreamsProcessor, Runnable
 
         ArrayNode tagArray = JsonNodeFactory.instance.arrayNode();
 
-        for( PercolateResponse.Match match : response.getMatches()) {
+        for (PercolateResponse.Match match : response.getMatches()) {
             tagArray.add(match.getId().string());
 
         }
@@ -131,14 +130,14 @@ public class PercolateProcessor implements StreamsProcessor, Runnable
     @Override
     public void run() {
 
-        while(true) {
+        while (true) {
             StreamsDatum item;
             try {
                 item = inQueue.poll();
 
                 Thread.sleep(new Random().nextInt(100));
 
-                for( StreamsDatum entry : process(item)) {
+                for (StreamsDatum entry : process(item)) {
                     outQueue.offer(entry);
                 }
 


[2/3] git commit: Refactored Elasticsearh components into their own class for better cohesion

Posted by mf...@apache.org.
Refactored Elasticsearh components into their own class for better cohesion


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b5fd7e70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b5fd7e70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b5fd7e70

Branch: refs/heads/STREAMS-58
Commit: b5fd7e70aa81515c4278cbaec788fbf3f81b8d58
Parents: af1efe8
Author: mfranklin <mf...@apache.org>
Authored: Tue Apr 22 11:34:39 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Apr 22 11:34:39 2014 -0400

----------------------------------------------------------------------
 .../ElasticsearchPersistReader.java             | 308 ++++---------------
 .../ElasticsearchPersistReaderTask.java         |  63 ----
 .../elasticsearch/ElasticsearchQuery.java       | 282 +++++++++++++++++
 3 files changed, 339 insertions(+), 314 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b5fd7e70/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
index 72a9954..fd2a155 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
@@ -1,25 +1,17 @@
 package org.apache.streams.elasticsearch;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Queues;
-import com.typesafe.config.Config;
-import org.apache.streams.config.StreamsConfigurator;
 import org.apache.streams.core.*;
 import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.sort.SortBuilders;
 import org.joda.time.DateTime;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.io.IOException;
+import java.io.Serializable;
 import java.math.BigInteger;
 import java.util.*;
 import java.util.concurrent.*;
@@ -32,95 +24,23 @@ import java.util.concurrent.*;
  * ************************************************************************************************************
  */
 
-public class ElasticsearchPersistReader implements StreamsPersistReader, Iterable<SearchHit>, Iterator<SearchHit> {
+public class ElasticsearchPersistReader implements StreamsPersistReader, Serializable {
     public static final String STREAMS_ID = "ElasticsearchPersistReader";
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
-    private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
-    private static final Integer DEFAULT_BATCH_SIZE = 500;
-    private static final String DEFAULT_SCROLL_TIMEOUT = "5m";
 
     protected volatile Queue<StreamsDatum> persistQueue;
 
-    private ElasticsearchClientManager elasticsearchClientManager;
-    private List<String> indexes = Lists.newArrayList();
-    private List<String> types = Lists.newArrayList();
-    private String[] withfields;
-    private String[] withoutfields;
-    private DateTime startDate;
-    private DateTime endDate;
-    private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
-    private boolean random = false;
+    private ElasticsearchQuery elasticsearchQuery;
+    private ElasticsearchReaderConfiguration config;
     private int threadPoolSize = 10;
-    private int batchSize = 100;
-    private String scrollTimeout = null;
-
-    private ObjectMapper mapper;
-
-    private ElasticsearchConfiguration config;
-
     private ExecutorService executor;
 
-    private QueryBuilder queryBuilder;
-    private FilterBuilder filterBuilder;
-
-    // These are private to help us manage the scroll
-    private SearchRequestBuilder search;
-    private SearchResponse scrollResp;
-    private int scrollPositionInScroll = SCROLL_POSITION_NOT_INITIALIZED;
-    private SearchHit next = null;
-    private long totalHits = 0;
-    private long totalRead = 0;
-
     public ElasticsearchPersistReader() {
-        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
-        this.config = ElasticsearchConfigurator.detectConfiguration(config);
-    }
-
-    public ElasticsearchPersistReader(ElasticsearchReaderConfiguration elasticsearchConfiguration) {
-        this.elasticsearchClientManager = new ElasticsearchClientManager(elasticsearchConfiguration);
-        indexes.addAll(elasticsearchConfiguration.getIndexes());
-        types.addAll(elasticsearchConfiguration.getTypes());
-    }
-
-    public long getHitCount() {
-        return this.search == null ? 0 : this.totalHits;
-    }
-
-    public long getReadCount() {
-        return this.totalRead;
-    }
-
-    public double getReadPercent() {
-        return (double) this.getReadCount() / (double) this.getHitCount();
-    }
-
-    public long getRemainingCount() {
-        return this.totalRead - this.totalHits;
-    }
-
-    public void setBatchSize(int batchSize) {
-        this.batchSize = batchSize;
     }
 
-    public void setScrollTimeout(String scrollTimeout) {
-        this.scrollTimeout = scrollTimeout;
-    }
-
-    public void setQueryBuilder(QueryBuilder queryBuilder) {
-        this.queryBuilder = queryBuilder;
-    }
-
-    public void setFilterBuilder(FilterBuilder filterBuilder) {
-        this.filterBuilder = filterBuilder;
-    }
-
-    public void setWithfields(String[] withfields) {
-        this.withfields = withfields;
-    }
-
-    public void setWithoutfields(String[] withoutfields) {
-        this.withoutfields = withoutfields;
+    public ElasticsearchPersistReader(ElasticsearchReaderConfiguration config) {
+        this.config = config;
     }
 
     //PersistReader methods
@@ -128,57 +48,14 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
     public void startStream() {
         LOGGER.debug("startStream");
         executor = Executors.newSingleThreadExecutor();
-        executor.submit(new ElasticsearchPersistReaderTask(this));
+        executor.submit(new ElasticsearchPersistReaderTask(this, elasticsearchQuery));
     }
 
     @Override
     public void prepare(Object o) {
-
-        mapper = StreamsJacksonMapper.getInstance();
-        persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
-
-        // If we haven't already set up the search, then set up the search.
-        if (search == null) {
-            search = elasticsearchClientManager.getClient()
-                    .prepareSearch(indexes.toArray(new String[0]))
-                    .setSearchType(SearchType.SCAN)
-                    .setSize(Objects.firstNonNull(batchSize, DEFAULT_BATCH_SIZE).intValue())
-                    .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT));
-
-            if (this.queryBuilder != null)
-                search.setQuery(this.queryBuilder);
-
-            // If the types are null, then don't specify a type
-            if (this.types != null && this.types.size() > 0)
-                search = search.setTypes(types.toArray(new String[0]));
-
-            Integer clauses = 0;
-            if (this.withfields != null || this.withoutfields != null) {
-                if (this.withfields != null)
-                    clauses += this.withfields.length;
-                if (this.withoutfields != null)
-                    clauses += this.withoutfields.length;
-            }
-
-            List<FilterBuilder> filterList = buildFilterList();
-
-            FilterBuilder allFilters = andFilters(filterList);
-
-            if (clauses > 0) {
-                //    search.setPostFilter(allFilters);
-                search.setPostFilter(allFilters);
-            }
-
-            // TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
-            if (this.random)
-                search = search.addSort(SortBuilders.scriptSort("random()", "number"));
-        }
-
-        // We don't have a scroll, we need to create a scroll
-        if (scrollResp == null) {
-            scrollResp = search.execute().actionGet();
-            LOGGER.trace(search.toString());
-        }
+        elasticsearchQuery = this.config == null ? new ElasticsearchQuery() : new ElasticsearchQuery(config);
+        elasticsearchQuery.execute(o);
+        persistQueue = constructQueue();
     }
 
     @Override
@@ -192,12 +69,12 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         StreamsResultSet current;
 
         synchronized (ElasticsearchPersistReader.class) {
-            current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(persistQueue));
+            current = new StreamsResultSet(persistQueue);
             current.setCounter(new DatumStatusCounter());
 //            current.getCounter().add(countersCurrent);
 //            countersTotal.add(countersCurrent);
 //            countersCurrent = new DatumStatusCounter();
-            persistQueue.clear();
+            persistQueue = constructQueue();
         }
 
         return current;
@@ -218,69 +95,20 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
 
     @Override
     public void cleanUp() {
+        this.shutdownAndAwaitTermination(executor);
         LOGGER.info("PersistReader done");
     }
 
-    //Iterable methods
-    @Override
-    public Iterator<SearchHit> iterator() {
-        return this;
-    }
-
-    //Iterator methods
-    @Override
-    public SearchHit next() {
-        return this.next;
-    }
-
-    @Override
-    public boolean hasNext() {
-        calcNext();
-        return hasRecords();
-    }
-
-
-    public void calcNext() {
-        try {
-            // We have exhausted our scroll create another scroll.
-            if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
-                // reset the scroll position
-                scrollPositionInScroll = 0;
-
-                // get the next hits of the scroll
-                scrollResp = elasticsearchClientManager.getClient()
-                        .prepareSearchScroll(scrollResp.getScrollId())
-                        .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT))
-                        .execute()
-                        .actionGet();
-
-                this.totalHits = scrollResp.getHits().getTotalHits();
-            }
-
-            // If this scroll has 0 items then we set the scroll position to -1
-            // letting the iterator know that we are done.
-            if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
-                scrollPositionInScroll = -1;
-            else {
-                // get the next record
-                next = scrollResp.getHits().getAt(scrollPositionInScroll);
-
-                // Increment our counters
-                scrollPositionInScroll += 1;
-                totalRead += 1;
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-            LOGGER.error("Unexpected scrolling error: {}", e.getMessage());
-            scrollPositionInScroll = -1;
-            next = null;
+    protected void write(StreamsDatum entry) {
+        boolean success;
+        do {
+            success = persistQueue.offer(entry);
+            Thread.yield();
         }
+        while (!success);
     }
 
-    public void remove() {
-    }
-
-    void shutdownAndAwaitTermination(ExecutorService pool) {
+    protected void shutdownAndAwaitTermination(ExecutorService pool) {
         pool.shutdown(); // Disable new tasks from being submitted
         try {
             // Wait a while for existing tasks to terminate
@@ -288,7 +116,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
                 pool.shutdownNow(); // Cancel currently executing tasks
                 // Wait a while for tasks to respond to being cancelled
                 if (!pool.awaitTermination(10, TimeUnit.SECONDS))
-                    System.err.println("Pool did not terminate");
+                    LOGGER.error("Pool did not terminate");
             }
         } catch (InterruptedException ie) {
             // (Re-)Cancel if current thread also interrupted
@@ -298,72 +126,50 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
         }
     }
 
-    private boolean isCompleted() {
-        return totalRead >= this.limit && hasRecords();
-    }
-
-    private boolean hasRecords() {
-        return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
-    }
-
-    // copied from elasticsearch
-    // if we need this again we should factor it out into a utility
-    private FilterBuilder andFilters(List<FilterBuilder> filters) {
-        if (filters == null || filters.size() == 0)
-            return null;
-
-        FilterBuilder toReturn = filters.get(0);
-
-        for (int i = 1; i < filters.size(); i++)
-            toReturn = FilterBuilders.andFilter(toReturn, filters.get(i));
-
-        return toReturn;
+    private Queue<StreamsDatum> constructQueue() {
+        return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
     }
 
-    private FilterBuilder orFilters(List<FilterBuilder> filters) {
-        if (filters == null || filters.size() == 0)
-            return null;
+    public static class ElasticsearchPersistReaderTask implements Runnable {
 
-        FilterBuilder toReturn = filters.get(0);
+        private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReaderTask.class);
 
-        for (int i = 1; i < filters.size(); i++)
-            toReturn = FilterBuilders.orFilter(toReturn, filters.get(i));
+        private ElasticsearchPersistReader reader;
+        private ElasticsearchQuery query;
+        private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
 
-        return toReturn;
-    }
-
-    private List<FilterBuilder> buildFilterList() {
-
-        ArrayList<FilterBuilder> filterList = Lists.newArrayList();
+        public ElasticsearchPersistReaderTask(ElasticsearchPersistReader reader, ElasticsearchQuery query) {
+            this.reader = reader;
+            this.query = query;
+        }
 
-        // If any withfields are specified, require that field be present
-        //    There must a value set also for the document to be processed
-        if (this.withfields != null && this.withfields.length > 0) {
-            ArrayList<FilterBuilder> withFilterList = Lists.newArrayList();
-            for (String withfield : this.withfields) {
-                FilterBuilder withFilter = FilterBuilders.existsFilter(withfield);
-                withFilterList.add(withFilter);
+        @Override
+        public void run() {
+
+            StreamsDatum item;
+            while (query.hasNext()) {
+                SearchHit hit = query.next();
+                ObjectNode jsonObject = null;
+                try {
+                    jsonObject = mapper.readValue(hit.getSourceAsString(), ObjectNode.class);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                    break;
+                }
+                item = new StreamsDatum(jsonObject, hit.getId());
+                item.getMetadata().put("id", hit.getId());
+                item.getMetadata().put("index", hit.getIndex());
+                item.getMetadata().put("type", hit.getType());
+                reader.write(item);
             }
-            //filterList.add(FilterBuilders.orFilter(orFilters(withFilterList)));
-            filterList.add(withFilterList.get(0));
-        }
-        // If any withoutfields are specified, require that field not be present
-        //    Document will be picked up even if present, if they do not have at least one value
-        // this is annoying as it majorly impacts runtime
-        // might be able to change behavior using null_field
-        if (this.withoutfields != null && this.withoutfields.length > 0) {
-            ArrayList<FilterBuilder> withoutFilterList = Lists.newArrayList();
-            for (String withoutfield : this.withoutfields) {
-                FilterBuilder withoutFilter = FilterBuilders.missingFilter(withoutfield).existence(true).nullValue(false);
-                withoutFilterList.add(withoutFilter);
+            try {
+                Thread.sleep(new Random().nextInt(100));
+            } catch (InterruptedException e) {
+                LOGGER.warn("Thread interrupted", e);
             }
-            //filterList.add(FilterBuilders.orFilter(orFilters(withoutFilterList)));
-            filterList.add(withoutFilterList.get(0));
-        }
 
-        return filterList;
+        }
     }
-
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b5fd7e70/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java
deleted file mode 100644
index 2d9c951..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.streams.elasticsearch;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-import org.elasticsearch.search.SearchHit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Queue;
-import java.util.Random;
-
-public class ElasticsearchPersistReaderTask implements Runnable {
-
-    private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReaderTask.class);
-
-    private ElasticsearchPersistReader reader;
-
-    private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
-    public ElasticsearchPersistReaderTask(ElasticsearchPersistReader reader) {
-        this.reader = reader;
-    }
-
-    @Override
-    public void run() {
-
-        StreamsDatum item;
-        while( reader.hasNext()) {
-            SearchHit hit = reader.next();
-            ObjectNode jsonObject = null;
-            try {
-                jsonObject = mapper.readValue(hit.getSourceAsString(), ObjectNode.class);
-            } catch (IOException e) {
-                e.printStackTrace();
-                break;
-            }
-            item = new StreamsDatum(jsonObject, hit.getId());
-            item.getMetadata().put("id", hit.getId());
-            item.getMetadata().put("index", hit.getIndex());
-            item.getMetadata().put("type", hit.getType());
-            write(item);
-        }
-        try {
-            Thread.sleep(new Random().nextInt(100));
-        } catch (InterruptedException e) {}
-
-    }
-
-    private void write( StreamsDatum entry ) {
-        boolean success;
-        do {
-            synchronized( ElasticsearchPersistReader.class ) {
-                success = reader.persistQueue.offer(entry);
-            }
-            Thread.yield();
-        }
-        while( !success );
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b5fd7e70/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
new file mode 100644
index 0000000..8c9abda
--- /dev/null
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
@@ -0,0 +1,282 @@
+package org.apache.streams.elasticsearch;
+
+import com.google.common.collect.Lists;
+import com.google.common.base.Objects;
+import com.typesafe.config.Config;
+import org.apache.streams.config.StreamsConfigurator;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ElasticsearchQuery implements Iterable<SearchHit>, Iterator<SearchHit>, Serializable {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchQuery.class);
+    private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
+    private static final Integer DEFAULT_BATCH_SIZE = 500;
+    private static final String DEFAULT_SCROLL_TIMEOUT = "5m";
+
+    private ElasticsearchClientManager elasticsearchClientManager;
+    private ElasticsearchConfiguration config;
+    private List<String> indexes = Lists.newArrayList();
+    private List<String> types = Lists.newArrayList();
+    private String[] withfields;
+    private String[] withoutfields;
+    private DateTime startDate;
+    private DateTime endDate;
+    private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
+    private boolean random = false;
+    private int batchSize = 100;
+    private String scrollTimeout = null;
+    private org.elasticsearch.index.query.QueryBuilder queryBuilder;
+    private org.elasticsearch.index.query.FilterBuilder filterBuilder;// These are private to help us manage the scroll
+    private SearchRequestBuilder search;
+    private SearchResponse scrollResp;
+    private int scrollPositionInScroll = SCROLL_POSITION_NOT_INITIALIZED;
+    private SearchHit next = null;
+    private long totalHits = 0;
+    private long totalRead = 0;
+
+    public ElasticsearchQuery() {
+        Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+        this.config = ElasticsearchConfigurator.detectConfiguration(config);
+    }
+
+    public ElasticsearchQuery(ElasticsearchReaderConfiguration config) {
+        this.config = config;
+        this.elasticsearchClientManager = new ElasticsearchClientManager(config);
+        this.indexes.addAll(config.getIndexes());
+        this.types.addAll(config.getTypes());
+    }
+
+    public long getHitCount() {
+        return this.search == null ? 0 : this.totalHits;
+    }
+
+    public long getReadCount() {
+        return this.totalRead;
+    }
+
+    public double getReadPercent() {
+        return (double) this.getReadCount() / (double) this.getHitCount();
+    }
+
+    public long getRemainingCount() {
+        return this.totalRead - this.totalHits;
+    }
+
+    public void setBatchSize(int batchSize) {
+        this.batchSize = batchSize;
+    }
+
+    public void setScrollTimeout(String scrollTimeout) {
+        this.scrollTimeout = scrollTimeout;
+    }
+
+    public void setQueryBuilder(QueryBuilder queryBuilder) {
+        this.queryBuilder = queryBuilder;
+    }
+
+    public void setFilterBuilder(FilterBuilder filterBuilder) {
+        this.filterBuilder = filterBuilder;
+    }
+
+    public void setWithfields(String[] withfields) {
+        this.withfields = withfields;
+    }
+
+    public void setWithoutfields(String[] withoutfields) {
+        this.withoutfields = withoutfields;
+    }
+
+    public void execute(Object o) {
+
+        // If we haven't already set up the search, then set up the search.
+        if (search == null) {
+            search = elasticsearchClientManager.getClient()
+                    .prepareSearch(indexes.toArray(new String[0]))
+                    .setSearchType(SearchType.SCAN)
+                    .setSize(Objects.firstNonNull(batchSize, DEFAULT_BATCH_SIZE).intValue())
+                    .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT));
+
+            if (this.queryBuilder != null)
+                search.setQuery(this.queryBuilder);
+
+            // If the types are null, then don't specify a type
+            if (this.types != null && this.types.size() > 0)
+                search = search.setTypes(types.toArray(new String[0]));
+
+            Integer clauses = 0;
+            if (this.withfields != null || this.withoutfields != null) {
+                if (this.withfields != null)
+                    clauses += this.withfields.length;
+                if (this.withoutfields != null)
+                    clauses += this.withoutfields.length;
+            }
+
+            List<FilterBuilder> filterList = buildFilterList();
+
+            FilterBuilder allFilters = andFilters(filterList);
+
+            if (clauses > 0) {
+                //    search.setPostFilter(allFilters);
+                search.setPostFilter(allFilters);
+            }
+
+            // TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
+            if (this.random)
+                search = search.addSort(SortBuilders.scriptSort("random()", "number"));
+        }
+
+        // We don't have a scroll, we need to create a scroll
+        if (scrollResp == null) {
+            scrollResp = search.execute().actionGet();
+            LOGGER.trace(search.toString());
+        }
+    }
+
+    //Iterable methods
+    @Override
+    public Iterator<SearchHit> iterator() {
+        return this;
+    }
+
+    //Iterator methods
+    @Override
+    public SearchHit next() {
+        return this.next;
+    }
+
+    @Override
+    public boolean hasNext() {
+        calcNext();
+        return hasRecords();
+    }
+
+    public void calcNext() {
+        try {
+            // We have exhausted our scroll create another scroll.
+            if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
+                // reset the scroll position
+                scrollPositionInScroll = 0;
+
+                // get the next hits of the scroll
+                scrollResp = elasticsearchClientManager.getClient()
+                        .prepareSearchScroll(scrollResp.getScrollId())
+                        .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT))
+                        .execute()
+                        .actionGet();
+
+                this.totalHits = scrollResp.getHits().getTotalHits();
+            }
+
+            // If this scroll has 0 items then we set the scroll position to -1
+            // letting the iterator know that we are done.
+            if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
+                scrollPositionInScroll = -1;
+            else {
+                // get the next record
+                next = scrollResp.getHits().getAt(scrollPositionInScroll);
+
+                // Increment our counters
+                scrollPositionInScroll += 1;
+                totalRead += 1;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            LOGGER.error("Unexpected scrolling error: {}", e.getMessage());
+            scrollPositionInScroll = -1;
+            next = null;
+        }
+    }
+
+    public void remove() {
+    }
+
+    protected boolean isCompleted() {
+        return totalRead >= this.limit && hasRecords();
+    }
+
+    protected boolean hasRecords() {
+        return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
+    }
+
+    // copied from elasticsearch
+    // if we need this again we should factor it out into a utility
+    private FilterBuilder andFilters(List<FilterBuilder> filters) {
+        if (filters == null || filters.size() == 0)
+            return null;
+
+        FilterBuilder toReturn = filters.get(0);
+
+        for (int i = 1; i < filters.size(); i++)
+            toReturn = FilterBuilders.andFilter(toReturn, filters.get(i));
+
+        return toReturn;
+    }
+
+    private FilterBuilder orFilters(List<FilterBuilder> filters) {
+        if (filters == null || filters.size() == 0)
+            return null;
+
+        FilterBuilder toReturn = filters.get(0);
+
+        for (int i = 1; i < filters.size(); i++)
+            toReturn = FilterBuilders.orFilter(toReturn, filters.get(i));
+
+        return toReturn;
+    }
+
+    private List<FilterBuilder> buildFilterList() {
+
+        // If any withfields are specified, require that field be present
+        //    There must a value set also for the document to be processed
+        // If any withoutfields are specified, require that field not be present
+        //    Document will be picked up even if present, if they do not have at least one value
+        // this is annoying as it majorly impacts runtime
+        // might be able to change behavior using null_field
+
+        ArrayList<FilterBuilder> filterList = Lists.newArrayList();
+
+        // If any withfields are specified, require that field be present
+        //    There must a value set also for the document to be processed
+        if (this.withfields != null && this.withfields.length > 0) {
+            ArrayList<FilterBuilder> withFilterList = Lists.newArrayList();
+            for (String withfield : this.withfields) {
+                FilterBuilder withFilter = FilterBuilders.existsFilter(withfield);
+                withFilterList.add(withFilter);
+            }
+            //filterList.add(FilterBuilders.orFilter(orFilters(withFilterList)));
+            filterList.add(withFilterList.get(0));
+        }
+        // If any withoutfields are specified, require that field not be present
+        //    Document will be picked up even if present, if they do not have at least one value
+        // this is annoying as it majorly impacts runtime
+        // might be able to change behavior using null_field
+        if (this.withoutfields != null && this.withoutfields.length > 0) {
+            ArrayList<FilterBuilder> withoutFilterList = Lists.newArrayList();
+            for (String withoutfield : this.withoutfields) {
+                FilterBuilder withoutFilter = FilterBuilders.missingFilter(withoutfield).existence(true).nullValue(false);
+                withoutFilterList.add(withoutFilter);
+            }
+            //filterList.add(FilterBuilders.orFilter(orFilters(withoutFilterList)));
+            filterList.add(withoutFilterList.get(0));
+        }
+
+        return filterList;
+    }
+}
\ No newline at end of file


[3/3] git commit: Updated locking to ensure multiple threads can offer to the queue but only one can swap the reference

Posted by mf...@apache.org.
Updated locking to ensure multiple threads can offer to the queue but only one can swap the reference


Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/f0085694
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/f0085694
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/f0085694

Branch: refs/heads/STREAMS-58
Commit: f00856946e6cb2ee83d05e96af29730d0f88dfe9
Parents: b5fd7e7
Author: mfranklin <mf...@apache.org>
Authored: Tue Apr 22 13:08:58 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Apr 22 13:08:58 2014 -0400

----------------------------------------------------------------------
 .../ElasticsearchPersistReader.java             | 20 +++++++++++++++++---
 .../elasticsearch/ElasticsearchQuery.java       |  2 --
 2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f0085694/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
index fd2a155..54d29d2 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
@@ -15,6 +15,8 @@ import java.io.Serializable;
 import java.math.BigInteger;
 import java.util.*;
 import java.util.concurrent.*;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * ***********************************************************************************************************
@@ -35,6 +37,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Seriali
     private ElasticsearchReaderConfiguration config;
     private int threadPoolSize = 10;
     private ExecutorService executor;
+    private ReadWriteLock lock = new ReentrantReadWriteLock();
 
     public ElasticsearchPersistReader() {
     }
@@ -68,13 +71,16 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Seriali
 
         StreamsResultSet current;
 
-        synchronized (ElasticsearchPersistReader.class) {
+        try {
+            lock.writeLock().lock();
             current = new StreamsResultSet(persistQueue);
             current.setCounter(new DatumStatusCounter());
 //            current.getCounter().add(countersCurrent);
 //            countersTotal.add(countersCurrent);
 //            countersCurrent = new DatumStatusCounter();
             persistQueue = constructQueue();
+        } finally {
+            lock.writeLock().unlock();
         }
 
         return current;
@@ -99,11 +105,19 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Seriali
         LOGGER.info("PersistReader done");
     }
 
+    //The locking may appear to be counter intuitive but we really don't care if multiple threads offer to the queue
+    //as it is a synchronized queue.  What we do care about is that we don't want to be offering to the current reference
+    //if the queue is being replaced with a new instance
     protected void write(StreamsDatum entry) {
         boolean success;
         do {
-            success = persistQueue.offer(entry);
-            Thread.yield();
+            try {
+                lock.readLock().lock();
+                success = persistQueue.offer(entry);
+                Thread.yield();
+            }finally {
+                lock.readLock().unlock();
+            }
         }
         while (!success);
     }

http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f0085694/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
index 8c9abda..2430c41 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
@@ -20,8 +20,6 @@ import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
 
 public class ElasticsearchQuery implements Iterable<SearchHit>, Iterator<SearchHit>, Serializable {