You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by mf...@apache.org on 2014/04/22 19:13:03 UTC
[1/3] git commit: Updated formatting and organization to better
reflect best practices
Repository: incubator-streams
Updated Branches:
refs/heads/STREAMS-58 [created] f00856946
Updated formatting and organization to better reflect best practices
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/af1efe8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/af1efe8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/af1efe8d
Branch: refs/heads/STREAMS-58
Commit: af1efe8df9a4d91359066757d229cceb19164369
Parents: 3d96497
Author: mfranklin <mf...@apache.org>
Authored: Tue Apr 22 10:04:56 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Apr 22 10:04:56 2014 -0400
----------------------------------------------------------------------
.../ElasticsearchClientManager.java | 152 +++----
.../ElasticsearchConfigurator.java | 4 +-
.../ElasticsearchPersistReader.java | 282 ++++++------
.../ElasticsearchPersistUpdater.java | 209 ++++-----
.../ElasticsearchPersistWriter.java | 440 +++++++++----------
.../elasticsearch/PercolateProcessor.java | 11 +-
6 files changed, 558 insertions(+), 540 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
index f1d6aad..2f8658a 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchClientManager.java
@@ -25,55 +25,110 @@ import java.util.concurrent.ExecutionException;
/**
* Created by sblackmon on 2/10/14.
*/
-public class ElasticsearchClientManager
-{
+public class ElasticsearchClientManager {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchClientManager.class);
private static Map<String, ElasticsearchClient> ALL_CLIENTS = new HashMap<String, ElasticsearchClient>();
private ElasticsearchConfiguration elasticsearchConfiguration;
- public ElasticsearchConfiguration getElasticsearchConfiguration() { return elasticsearchConfiguration; }
- public String toString() { return ToStringBuilder.reflectionToString(this); }
- public boolean equals(Object o) { return EqualsBuilder.reflectionEquals(this, o, Arrays.asList(this.elasticsearchConfiguration.toString())); }
- public int hashCode() { return HashCodeBuilder.reflectionHashCode(this, Arrays.asList(this.elasticsearchConfiguration.toString())); }
-
public ElasticsearchClientManager(ElasticsearchConfiguration elasticsearchConfiguration) {
this.elasticsearchConfiguration = elasticsearchConfiguration;
}
- /**************************************************************************************
+ public ElasticsearchConfiguration getElasticsearchConfiguration() {
+ return elasticsearchConfiguration;
+ }
+
+ /**
+ * ***********************************************************************************
* Get the Client for this return, it is actually a transport client, but it is much
* easier to work with the generic object as this interface likely won't change from
* elasticsearch. This method is synchronized to block threads from creating
* too many of these at any given time.
- * @return
- * Client for elasticsearch
- *************************************************************************************/
- public Client getClient()
- {
+ *
+ * @return Client for elasticsearch
+ * ***********************************************************************************
+ */
+ public Client getClient() {
checkAndLoadClient(null);
return ALL_CLIENTS.get(this.elasticsearchConfiguration.getClusterName()).getClient();
}
- public Client getClient(String clusterName)
- {
+ public Client getClient(String clusterName) {
checkAndLoadClient(clusterName);
return ALL_CLIENTS.get(this.elasticsearchConfiguration.getClusterName()).getClient();
}
+ public boolean isOnOrAfterVersion(Version version) {
+ return ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getVersion().onOrAfter(version);
+ }
+
+ public void start() throws Exception {
+ /***********************************************************************
+ * Note:
+ * Everything in these classes is being switched to lazy loading. Within
+ * Heroku you only have 60 seconds to connect, and bind to the service,
+ * and you are only allowed to run in 1Gb of memory. Switching all
+ * of this to lazy loading is how we are fixing some of the issues
+ * if you are having issues with these classes, please, refactor
+ * and create a UNIT TEST CASE!!!!!! To ensure that everything is
+ * working before you check it back in.
+ *
+ * Author: Smashew @ 2013-08-26
+ **********************************************************************/
+ }
+
+ public boolean refresh(String index) {
+ return refresh(new String[]{index});
+ }
+
+ public boolean refresh(String[] indexes) {
+ RefreshResponse refreshResponse = this.getClient().admin().indices().prepareRefresh(indexes).execute().actionGet();
+ return refreshResponse.getFailedShards() == 0;
+ }
+
+ public synchronized void stop() {
+ // Terminate the elasticsearch cluster
+ // Check to see if we have a client.
+ if (ALL_CLIENTS.containsKey(this.elasticsearchConfiguration.toString())) {
+ // Close the client
+ ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getClient().close();
+
+ // Remove it so that it isn't in memory any more.
+ ALL_CLIENTS.remove(this.elasticsearchConfiguration.toString());
+ }
+ }
+
+ public ClusterHealthResponse getStatus() throws ExecutionException, InterruptedException {
+ return new ClusterHealthRequestBuilder(this.getClient().admin().cluster())
+ .execute()
+ .get();
+ }
+
+ public String toString() {
+ return ToStringBuilder.reflectionToString(this);
+ }
+
+ public boolean equals(Object o) {
+ return EqualsBuilder.reflectionEquals(this, o, Arrays.asList(this.elasticsearchConfiguration.toString()));
+ }
+
+ public int hashCode() {
+ return HashCodeBuilder.reflectionHashCode(this, Arrays.asList(this.elasticsearchConfiguration.toString()));
+ }
+
private synchronized void checkAndLoadClient(String clusterName) {
- if( clusterName == null )
+ if (clusterName == null)
clusterName = this.elasticsearchConfiguration.getClusterName();
// If it is there, exit early
if (ALL_CLIENTS.containsKey(clusterName))
return;
- try
- {
+ try {
// We are currently using lazy loading to start the elasticsearch cluster, however.
LOGGER.info("Creating a new TransportClient: {}", this.elasticsearchConfiguration.getHosts());
@@ -86,7 +141,7 @@ public class ElasticsearchClientManager
// Create the client
TransportClient client = new TransportClient(settings);
- for(String h : this.getElasticsearchConfiguration().getHosts()) {
+ for (String h : this.getElasticsearchConfiguration().getHosts()) {
LOGGER.info("Adding Host: {}", h);
client.addTransportAddress(new InetSocketTransportAddress(h, this.getElasticsearchConfiguration().getPort().intValue()));
}
@@ -97,75 +152,20 @@ public class ElasticsearchClientManager
// Add it to our static map
ALL_CLIENTS.put(clusterName, elasticsearchClient);
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
LOGGER.error("Could not Create elasticsearch Transport Client: {}", e);
}
}
-
private Version getVersion(Client client) {
try {
ClusterStateRequestBuilder clusterStateRequestBuilder = new ClusterStateRequestBuilder(client.admin().cluster());
ClusterStateResponse clusterStateResponse = clusterStateRequestBuilder.execute().actionGet();
return clusterStateResponse.getState().getNodes().getMasterNode().getVersion();
- }
- catch (Exception e) {
+ } catch (Exception e) {
return null;
}
}
-
-
- public boolean isOnOrAfterVersion(Version version) {
- return ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getVersion().onOrAfter(version);
- }
-
- public void start() throws Exception
- {
- /***********************************************************************
- * Note:
- * Everything in these classes is being switched to lazy loading. Within
- * Heroku you only have 60 seconds to connect, and bind to the service,
- * and you are only allowed to run in 1Gb of memory. Switching all
- * of this to lazy loading is how we are fixing some of the issues
- * if you are having issues with these classes, please, refactor
- * and create a UNIT TEST CASE!!!!!! To ensure that everything is
- * working before you check it back in.
- *
- * Author: Smashew @ 2013-08-26
- **********************************************************************/
- }
-
- public boolean refresh(String index) {
- return refresh(new String[]{index});
- }
-
- public boolean refresh(String[] indexes) {
- RefreshResponse refreshResponse = this.getClient().admin().indices().prepareRefresh(indexes).execute().actionGet();
- return refreshResponse.getFailedShards() == 0;
- }
-
- public synchronized void stop()
- {
- // Terminate the elasticsearch cluster
- // Check to see if we have a client.
- if (ALL_CLIENTS.containsKey(this.elasticsearchConfiguration.toString()))
- {
- // Close the client
- ALL_CLIENTS.get(this.elasticsearchConfiguration.toString()).getClient().close();
-
- // Remove it so that it isn't in memory any more.
- ALL_CLIENTS.remove(this.elasticsearchConfiguration.toString());
- }
- }
-
- public ClusterHealthResponse getStatus() throws ExecutionException, InterruptedException
- {
- return new ClusterHealthRequestBuilder(this.getClient().admin().cluster())
- .execute()
- .get();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
index 20b5c08..483a177 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchConfigurator.java
@@ -33,7 +33,7 @@ public class ElasticsearchConfigurator {
public static ElasticsearchReaderConfiguration detectReaderConfiguration(Config elasticsearch) {
ElasticsearchConfiguration elasticsearchConfiguration = detectConfiguration(elasticsearch);
- ElasticsearchReaderConfiguration elasticsearchReaderConfiguration = mapper.convertValue(elasticsearchConfiguration, ElasticsearchReaderConfiguration.class);
+ ElasticsearchReaderConfiguration elasticsearchReaderConfiguration = mapper.convertValue(elasticsearchConfiguration, ElasticsearchReaderConfiguration.class);
List<String> indexes = elasticsearch.getStringList("indexes");
List<String> types = elasticsearch.getStringList("types");
@@ -47,7 +47,7 @@ public class ElasticsearchConfigurator {
public static ElasticsearchWriterConfiguration detectWriterConfiguration(Config elasticsearch) {
ElasticsearchConfiguration elasticsearchConfiguration = detectConfiguration(elasticsearch);
- ElasticsearchWriterConfiguration elasticsearchWriterConfiguration = mapper.convertValue(elasticsearchConfiguration, ElasticsearchWriterConfiguration.class);
+ ElasticsearchWriterConfiguration elasticsearchWriterConfiguration = mapper.convertValue(elasticsearchConfiguration, ElasticsearchWriterConfiguration.class);
String index = elasticsearch.getString("index");
String type = elasticsearch.getString("type");
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
index ea74bf6..72a9954 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
@@ -1,14 +1,9 @@
package org.apache.streams.elasticsearch;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Objects;
-import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.collect.Queues;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
import com.typesafe.config.Config;
import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.*;
@@ -25,29 +20,28 @@ import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.math.BigInteger;
import java.util.*;
import java.util.concurrent.*;
-/**************************************************************************************************************
+/**
+ * ***********************************************************************************************************
* Authors:
* smashew
* steveblackmon
- **************************************************************************************************************/
+ * ************************************************************************************************************
+ */
-public class ElasticsearchPersistReader implements StreamsPersistReader, Iterable<SearchHit>, Iterator<SearchHit>
-{
- public final static String STREAMS_ID = "ElasticsearchPersistReader";
-
- private final static Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
-
- protected volatile Queue<StreamsDatum> persistQueue;
+public class ElasticsearchPersistReader implements StreamsPersistReader, Iterable<SearchHit>, Iterator<SearchHit> {
+ public static final String STREAMS_ID = "ElasticsearchPersistReader";
+ private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
private static final Integer DEFAULT_BATCH_SIZE = 500;
private static final String DEFAULT_SCROLL_TIMEOUT = "5m";
+ protected volatile Queue<StreamsDatum> persistQueue;
+
private ElasticsearchClientManager elasticsearchClientManager;
private List<String> indexes = Lists.newArrayList();
private List<String> types = Lists.newArrayList();
@@ -55,7 +49,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
private String[] withoutfields;
private DateTime startDate;
private DateTime endDate;
- private int limit = 1000*1000*1000; // we are going to set the default limit very high to 1bil
+ private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
private boolean random = false;
private int threadPoolSize = 10;
private int batchSize = 100;
@@ -78,14 +72,32 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
private long totalHits = 0;
private long totalRead = 0;
- public long getHitCount() { return this.search == null ? 0 : this.totalHits; }
- public long getReadCount() { return this.totalRead; }
- public double getReadPercent() { return (double)this.getReadCount() / (double)this.getHitCount(); }
- public long getRemainingCount() { return this.totalRead - this.totalHits; }
- public Iterator<SearchHit> iterator() { return this; }
- private boolean isCompleted() { return totalRead >= this.limit && hasRecords(); }
- private boolean hasRecords() { return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit)); }
- public SearchHit next() { return this.next; }
+ public ElasticsearchPersistReader() {
+ Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+ this.config = ElasticsearchConfigurator.detectConfiguration(config);
+ }
+
+ public ElasticsearchPersistReader(ElasticsearchReaderConfiguration elasticsearchConfiguration) {
+ this.elasticsearchClientManager = new ElasticsearchClientManager(elasticsearchConfiguration);
+ indexes.addAll(elasticsearchConfiguration.getIndexes());
+ types.addAll(elasticsearchConfiguration.getTypes());
+ }
+
+ public long getHitCount() {
+ return this.search == null ? 0 : this.totalHits;
+ }
+
+ public long getReadCount() {
+ return this.totalRead;
+ }
+
+ public double getReadPercent() {
+ return (double) this.getReadCount() / (double) this.getHitCount();
+ }
+
+ public long getRemainingCount() {
+ return this.totalRead - this.totalHits;
+ }
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
@@ -95,19 +107,23 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
this.scrollTimeout = scrollTimeout;
}
- public void setQueryBuilder(QueryBuilder queryBuilder) { this.queryBuilder = queryBuilder; }
- public void setFilterBuilder(FilterBuilder filterBuilder) { this.filterBuilder = filterBuilder; }
+ public void setQueryBuilder(QueryBuilder queryBuilder) {
+ this.queryBuilder = queryBuilder;
+ }
- public ElasticsearchPersistReader() {
- Config config = StreamsConfigurator.config.getConfig("elasticsearch");
- this.config = ElasticsearchConfigurator.detectConfiguration(config);
+ public void setFilterBuilder(FilterBuilder filterBuilder) {
+ this.filterBuilder = filterBuilder;
}
- public ElasticsearchPersistReader(ElasticsearchReaderConfiguration elasticsearchConfiguration) {
- this.elasticsearchClientManager = new ElasticsearchClientManager(elasticsearchConfiguration);
- indexes.addAll(elasticsearchConfiguration.getIndexes());
- types.addAll(elasticsearchConfiguration.getTypes());
+
+ public void setWithfields(String[] withfields) {
+ this.withfields = withfields;
+ }
+
+ public void setWithoutfields(String[] withoutfields) {
+ this.withoutfields = withoutfields;
}
+ //PersistReader methods
@Override
public void startStream() {
LOGGER.debug("startStream");
@@ -122,26 +138,25 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
// If we haven't already set up the search, then set up the search.
- if(search == null)
- {
+ if (search == null) {
search = elasticsearchClientManager.getClient()
.prepareSearch(indexes.toArray(new String[0]))
.setSearchType(SearchType.SCAN)
.setSize(Objects.firstNonNull(batchSize, DEFAULT_BATCH_SIZE).intValue())
.setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT));
- if(this.queryBuilder != null)
+ if (this.queryBuilder != null)
search.setQuery(this.queryBuilder);
// If the types are null, then don't specify a type
- if(this.types != null && this.types.size() > 0)
+ if (this.types != null && this.types.size() > 0)
search = search.setTypes(types.toArray(new String[0]));
Integer clauses = 0;
- if(this.withfields != null || this.withoutfields != null) {
- if( this.withfields != null )
+ if (this.withfields != null || this.withoutfields != null) {
+ if (this.withfields != null)
clauses += this.withfields.length;
- if( this.withoutfields != null )
+ if (this.withoutfields != null)
clauses += this.withoutfields.length;
}
@@ -149,44 +164,86 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
FilterBuilder allFilters = andFilters(filterList);
- if( clauses > 0 ) {
+ if (clauses > 0) {
// search.setPostFilter(allFilters);
search.setPostFilter(allFilters);
}
// TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
- if(this.random)
+ if (this.random)
search = search.addSort(SortBuilders.scriptSort("random()", "number"));
}
// We don't have a scroll, we need to create a scroll
- if(scrollResp == null) {
+ if (scrollResp == null) {
scrollResp = search.execute().actionGet();
LOGGER.trace(search.toString());
}
}
- public void setWithfields(String[] withfields) {
- this.withfields = withfields;
+ @Override
+ public StreamsResultSet readAll() {
+ return readCurrent();
}
- public void setWithoutfields(String[] withoutfields) {
- this.withoutfields = withoutfields;
+ @Override
+ public StreamsResultSet readCurrent() {
+
+ StreamsResultSet current;
+
+ synchronized (ElasticsearchPersistReader.class) {
+ current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(persistQueue));
+ current.setCounter(new DatumStatusCounter());
+// current.getCounter().add(countersCurrent);
+// countersTotal.add(countersCurrent);
+// countersCurrent = new DatumStatusCounter();
+ persistQueue.clear();
+ }
+
+ return current;
+
+ }
+
+ //TODO - This just reads current records and does not adjust any queries
+ @Override
+ public StreamsResultSet readNew(BigInteger sequence) {
+ return readCurrent();
+ }
+
+ //TODO - This just reads current records and does not adjust any queries
+ @Override
+ public StreamsResultSet readRange(DateTime start, DateTime end) {
+ return readCurrent();
+ }
+
+ @Override
+ public void cleanUp() {
+ LOGGER.info("PersistReader done");
+ }
+
+ //Iterable methods
+ @Override
+ public Iterator<SearchHit> iterator() {
+ return this;
+ }
+
+ //Iterator methods
+ @Override
+ public SearchHit next() {
+ return this.next;
}
- public boolean hasNext()
- {
+ @Override
+ public boolean hasNext() {
calcNext();
return hasRecords();
}
- public void calcNext()
- {
- try
- {
+
+ public void calcNext() {
+ try {
// We have exhausted our scroll create another scroll.
- if(scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length)
- {
+ if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
// reset the scroll position
scrollPositionInScroll = 0;
@@ -202,10 +259,9 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
// If this scroll has 0 items then we set the scroll position to -1
// letting the iterator know that we are done.
- if(scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
+ if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
scrollPositionInScroll = -1;
- else
- {
+ else {
// get the next record
next = scrollResp.getHits().getAt(scrollPositionInScroll);
@@ -213,9 +269,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
scrollPositionInScroll += 1;
totalRead += 1;
}
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
e.printStackTrace();
LOGGER.error("Unexpected scrolling error: {}", e.getMessage());
scrollPositionInScroll = -1;
@@ -223,31 +277,56 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
}
}
- public void remove() { }
+ public void remove() {
+ }
+
+ void shutdownAndAwaitTermination(ExecutorService pool) {
+ pool.shutdown(); // Disable new tasks from being submitted
+ try {
+ // Wait a while for existing tasks to terminate
+ if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
+ pool.shutdownNow(); // Cancel currently executing tasks
+ // Wait a while for tasks to respond to being cancelled
+ if (!pool.awaitTermination(10, TimeUnit.SECONDS))
+ System.err.println("Pool did not terminate");
+ }
+ } catch (InterruptedException ie) {
+ // (Re-)Cancel if current thread also interrupted
+ pool.shutdownNow();
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ private boolean isCompleted() {
+ return totalRead >= this.limit && hasRecords();
+ }
+
+ private boolean hasRecords() {
+ return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
+ }
// copied from elasticsearch
// if we need this again we should factor it out into a utility
- private FilterBuilder andFilters(List<FilterBuilder> filters)
- {
- if(filters == null || filters.size() == 0)
+ private FilterBuilder andFilters(List<FilterBuilder> filters) {
+ if (filters == null || filters.size() == 0)
return null;
FilterBuilder toReturn = filters.get(0);
- for(int i = 1; i < filters.size(); i++)
+ for (int i = 1; i < filters.size(); i++)
toReturn = FilterBuilders.andFilter(toReturn, filters.get(i));
return toReturn;
}
- private FilterBuilder orFilters(List<FilterBuilder> filters)
- {
- if(filters == null || filters.size() == 0)
+ private FilterBuilder orFilters(List<FilterBuilder> filters) {
+ if (filters == null || filters.size() == 0)
return null;
FilterBuilder toReturn = filters.get(0);
- for(int i = 1; i < filters.size(); i++)
+ for (int i = 1; i < filters.size(); i++)
toReturn = FilterBuilders.orFilter(toReturn, filters.get(i));
return toReturn;
@@ -259,9 +338,9 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
// If any withfields are specified, require that field be present
// There must a value set also for the document to be processed
- if(this.withfields != null && this.withfields.length > 0) {
+ if (this.withfields != null && this.withfields.length > 0) {
ArrayList<FilterBuilder> withFilterList = Lists.newArrayList();
- for( String withfield : this.withfields ) {
+ for (String withfield : this.withfields) {
FilterBuilder withFilter = FilterBuilders.existsFilter(withfield);
withFilterList.add(withFilter);
}
@@ -272,9 +351,9 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
// Document will be picked up even if present, if they do not have at least one value
// this is annoying as it majorly impacts runtime
// might be able to change behavior using null_field
- if(this.withoutfields != null && this.withoutfields.length > 0) {
+ if (this.withoutfields != null && this.withoutfields.length > 0) {
ArrayList<FilterBuilder> withoutFilterList = Lists.newArrayList();
- for( String withoutfield : this.withoutfields ) {
+ for (String withoutfield : this.withoutfields) {
FilterBuilder withoutFilter = FilterBuilders.missingFilter(withoutfield).existence(true).nullValue(false);
withoutFilterList.add(withoutFilter);
}
@@ -285,61 +364,6 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
return filterList;
}
- @Override
- public StreamsResultSet readCurrent() {
-
- StreamsResultSet current;
-
- synchronized( ElasticsearchPersistReader.class ) {
- current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(persistQueue));
- current.setCounter(new DatumStatusCounter());
-// current.getCounter().add(countersCurrent);
-// countersTotal.add(countersCurrent);
-// countersCurrent = new DatumStatusCounter();
- persistQueue.clear();
- }
-
- return current;
-
- }
-
- public StreamsResultSet readAll() {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readNew(BigInteger sequence) {
- return readCurrent();
- }
-
- @Override
- public StreamsResultSet readRange(DateTime start, DateTime end) {
- return readCurrent();
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
- pool.shutdown(); // Disable new tasks from being submitted
- try {
- // Wait a while for existing tasks to terminate
- if (!pool.awaitTermination(10, TimeUnit.SECONDS)) {
- pool.shutdownNow(); // Cancel currently executing tasks
- // Wait a while for tasks to respond to being cancelled
- if (!pool.awaitTermination(10, TimeUnit.SECONDS))
- System.err.println("Pool did not terminate");
- }
- } catch (InterruptedException ie) {
- // (Re-)Cancel if current thread also interrupted
- pool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
- @Override
- public void cleanUp() {
- LOGGER.info("PersistReader done");
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
index 5f59950..662a084 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistUpdater.java
@@ -41,8 +41,7 @@ import java.util.*;
//import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequest;
-public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable
-{
+public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable {
private final static Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistUpdater.class);
private final static NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
private final static NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
@@ -92,19 +91,53 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
private final List<String> affectedIndexes = new ArrayList<String>();
- public int getTotalOutstanding() { return this.totalSent - (this.totalFailed + this.totalOk); }
- public long getFlushThresholdSizeInBytes() { return flushThresholdSizeInBytes; }
- public int getTotalSent() { return totalSent; }
- public int getTotalSeconds() { return totalSeconds; }
- public int getTotalOk() { return totalOk; }
- public int getTotalFailed() { return totalFailed; }
- public int getTotalBatchCount() { return totalBatchCount; }
- public long getTotalSizeInBytes() { return totalSizeInBytes; }
- public long getBatchSizeInBytes() { return batchSizeInBytes; }
- public int getBatchItemsSent() { return batchItemsSent; }
- public List<String> getAffectedIndexes() { return this.affectedIndexes; }
+ public int getTotalOutstanding() {
+ return this.totalSent - (this.totalFailed + this.totalOk);
+ }
+
+ public long getFlushThresholdSizeInBytes() {
+ return flushThresholdSizeInBytes;
+ }
+
+ public int getTotalSent() {
+ return totalSent;
+ }
+
+ public int getTotalSeconds() {
+ return totalSeconds;
+ }
+
+ public int getTotalOk() {
+ return totalOk;
+ }
+
+ public int getTotalFailed() {
+ return totalFailed;
+ }
+
+ public int getTotalBatchCount() {
+ return totalBatchCount;
+ }
+
+ public long getTotalSizeInBytes() {
+ return totalSizeInBytes;
+ }
+
+ public long getBatchSizeInBytes() {
+ return batchSizeInBytes;
+ }
+
+ public int getBatchItemsSent() {
+ return batchItemsSent;
+ }
+
+ public List<String> getAffectedIndexes() {
+ return this.affectedIndexes;
+ }
- public void setFlushThresholdSizeInBytes(long sizeInBytes) { this.flushThresholdSizeInBytes = sizeInBytes; }
+ public void setFlushThresholdSizeInBytes(long sizeInBytes) {
+ this.flushThresholdSizeInBytes = sizeInBytes;
+ }
Thread task;
@@ -123,12 +156,14 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
this.config = config;
}
- private static final int BYTES_IN_MB = 1024*1024;
- private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
- private volatile int totalByteCount = 0;
- private volatile int byteCount = 0;
+ private static final int BYTES_IN_MB = 1024 * 1024;
+ private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+ private volatile int totalByteCount = 0;
+ private volatile int byteCount = 0;
- public boolean isConnected() { return (client != null); }
+ public boolean isConnected() {
+ return (client != null);
+ }
@Override
public void write(StreamsDatum streamsDatum) {
@@ -173,30 +208,25 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
}
@Override
- public void close()
- {
- try
- {
+ public void close() {
+ try {
// before they close, check to ensure that
this.flush();
int count = 0;
// We are going to give it 5 minutes.
- while(this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
+ while (this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
Thread.sleep(50);
- if(this.getTotalOutstanding() > 0)
- {
+ if (this.getTotalOutstanding() > 0) {
LOGGER.error("We never cleared our buffer");
}
- for(String indexName : this.getAffectedIndexes())
- {
+ for (String indexName : this.getAffectedIndexes()) {
createIndexIfMissing(indexName);
- if(this.veryLargeBulk)
- {
+ if (this.veryLargeBulk) {
LOGGER.debug("Resetting our Refresh Interval: {}", indexName);
// They are in 'very large bulk' mode and the process is finished. We now want to turn the
// refreshing back on.
@@ -224,9 +254,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
LOGGER.info("Closed: Wrote[{} of {}] Failed[{}]", this.getTotalOk(), this.getTotalSent(), this.getTotalFailed());
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
// this line of code should be logically unreachable.
LOGGER.warn("This is unexpected: {}", e.getMessage());
e.printStackTrace();
@@ -234,17 +262,14 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
}
@Override
- public void flush() throws IOException
- {
+ public void flush() throws IOException {
flushInternal();
}
- public void flushInternal()
- {
- synchronized (this)
- {
+ public void flushInternal() {
+ synchronized (this) {
// we do not have a working bulk request, we can just exit here.
- if(this.bulkRequest == null || batchItemsSent == 0)
+ if (this.bulkRequest == null || batchItemsSent == 0)
return;
// call the flush command.
@@ -261,11 +286,9 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
this.batchSizeInBytes = 0;
this.batchItemsSent = 0;
- try
- {
+ try {
int count = 0;
- if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
- {
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
/****************************************************************************
* Author:
* Smashew
@@ -290,27 +313,22 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
****************************************************************************/
// wait for the flush to catch up. We are going to cap this at
- while(this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
+ while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
Thread.sleep(10);
- if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
}
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
LOGGER.info("We were broken from our loop: {}", e.getMessage());
}
}
}
- private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes)
- {
- bulkRequest.execute().addListener(new ActionListener<BulkResponse>()
- {
+ private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+ bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
@Override
- public void onResponse(BulkResponse bulkItemResponses)
- {
+ public void onResponse(BulkResponse bulkItemResponses) {
if (bulkItemResponses.hasFailures())
LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
@@ -319,9 +337,8 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
long thisMillis = bulkItemResponses.getTookInMillis();
// keep track of the number of totalFailed and items that we have totalOk.
- for(BulkItemResponse resp : bulkItemResponses.getItems())
- {
- if(resp.isFailed())
+ for (BulkItemResponse resp : bulkItemResponses.getItems()) {
+ if (resp.isFailed())
thisFailed++;
else
thisOk++;
@@ -331,17 +348,16 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
totalFailed += thisFailed;
totalSeconds += (thisMillis / 1000);
- if(thisSent != (thisOk + thisFailed))
+ if (thisSent != (thisOk + thisFailed))
LOGGER.error("We sent more items than this");
LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
- MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
- MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+ MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
+ MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
}
@Override
- public void onFailure(Throwable e)
- {
+ public void onFailure(Throwable e) {
LOGGER.error("Error bulk loading: {}", e.getMessage());
e.printStackTrace();
}
@@ -350,8 +366,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
this.notify();
}
- public void add(String indexName, String type, String id, String json)
- {
+ public void add(String indexName, String type, String id, String json) {
UpdateRequest updateRequest;
Preconditions.checkNotNull(id);
@@ -359,20 +374,18 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
// They didn't specify an ID, so we will create one for them.
updateRequest = new UpdateRequest()
- .index(indexName)
- .type(type)
- .id(id)
- .doc(json);
+ .index(indexName)
+ .type(type)
+ .id(id)
+ .doc(json);
add(updateRequest);
}
- public void add(UpdateRequest updateRequest)
- {
+ public void add(UpdateRequest updateRequest) {
Preconditions.checkNotNull(updateRequest);
- synchronized (this)
- {
+ synchronized (this) {
checkAndCreateBulkRequest();
checkIndexImplications(updateRequest.index());
bulkRequest.add(updateRequest);
@@ -381,37 +394,33 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
Optional.fromNullable(updateRequest.doc().source().length()),
Optional.fromNullable(updateRequest.script().length()));
trackItemAndBytesWritten(size.get().longValue());
- } catch( NullPointerException x) {
+ } catch (NullPointerException x) {
trackItemAndBytesWritten(1000);
}
}
}
- private void trackItemAndBytesWritten(long sizeInBytes)
- {
+ private void trackItemAndBytesWritten(long sizeInBytes) {
batchItemsSent++;
batchSizeInBytes += sizeInBytes;
// If our queue is larger than our flush threashold, then we should flush the queue.
- if(batchSizeInBytes > flushThresholdSizeInBytes)
+ if (batchSizeInBytes > flushThresholdSizeInBytes)
flushInternal();
}
- private void checkAndCreateBulkRequest()
- {
+ private void checkAndCreateBulkRequest() {
// Synchronize to ensure that we don't lose any records
- synchronized (this)
- {
- if(bulkRequest == null)
+ synchronized (this) {
+ if (bulkRequest == null)
bulkRequest = this.manager.getClient().prepareBulk();
}
}
- private void checkIndexImplications(String indexName)
- {
+ private void checkIndexImplications(String indexName) {
// check to see if we have seen this index before.
- if(this.affectedIndexes.contains(indexName))
+ if (this.affectedIndexes.contains(indexName))
return;
// we haven't log this index.
@@ -419,7 +428,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
// Check to see if we are in 'veryLargeBulk' mode
// if we aren't, exit early
- if(!this.veryLargeBulk)
+ if (!this.veryLargeBulk)
return;
@@ -438,24 +447,20 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
}
public void createIndexIfMissing(String indexName) {
- if(!this.manager.getClient()
+ if (!this.manager.getClient()
.admin()
.indices()
.exists(new IndicesExistsRequest(indexName))
.actionGet()
- .isExists())
- {
+ .isExists()) {
// It does not exist... So we are going to need to create the index.
// we are going to assume that the 'templates' that we have loaded into
// elasticsearch are sufficient to ensure the index is being created properly.
CreateIndexResponse response = this.manager.getClient().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
- if(response.isAcknowledged())
- {
+ if (response.isAcknowledged()) {
LOGGER.info("Index {} did not exist. The index was automatically created from the stored ElasticSearch Templates.", indexName);
- }
- else
- {
+ } else {
LOGGER.error("Index {} did not exist. While attempting to create the index from stored ElasticSearch Templates we were unable to get an acknowledgement.", indexName);
LOGGER.error("Error Message: {}", response.toString());
throw new RuntimeException("Unable to create index " + indexName);
@@ -463,18 +468,16 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
}
}
- public void add(String indexName, String type, Map<String, Object> toImport)
- {
+ public void add(String indexName, String type, Map<String, Object> toImport) {
for (String id : toImport.keySet())
- add(indexName, type, id, (String)toImport.get(id));
+ add(indexName, type, id, (String) toImport.get(id));
}
- private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch)
- {
+ private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch) {
Set<String> invalidIDs = checkIds(workingBatch.keySet(), index, type);
- for(String toAddId : workingBatch.keySet())
- if(!invalidIDs.contains(toAddId))
+ for (String toAddId : workingBatch.keySet())
+ if (!invalidIDs.contains(toAddId))
add(index, type, toAddId, workingBatch.get(toAddId));
LOGGER.info("Adding Batch: {} -> {}", workingBatch.size(), invalidIDs.size());
@@ -485,7 +488,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
IdsQueryBuilder idsFilterBuilder = new IdsQueryBuilder();
- for(String s : input)
+ for (String s : input)
idsFilterBuilder.addIds(s);
SearchRequestBuilder searchRequestBuilder = this.manager.getClient()
@@ -501,7 +504,7 @@ public class ElasticsearchPersistUpdater extends ElasticsearchPersistWriter impl
Set<String> toReturn = new HashSet<String>();
- for(SearchHit hit : hits) {
+ for (SearchHit hit : hits) {
toReturn.add(hit.getId());
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
index ad89013..ff7e0f5 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistWriter.java
@@ -37,28 +37,35 @@ import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
-public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable
-{
- public final static String STREAMS_ID = "ElasticsearchPersistWriter";
+public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushable, Closeable, DatumStatusCountable {
+ public static final String STREAMS_ID = "ElasticsearchPersistWriter";
- private final static Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
- private final static NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
- private final static NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
+ public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistWriter.class);
+ private static final NumberFormat MEGABYTE_FORMAT = new DecimalFormat("#.##");
+ private static final NumberFormat NUMBER_FORMAT = new DecimalFormat("###,###,###,###");
+ private static final Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
+ private static final long WAITING_DOCS_LIMIT = 10000;
+ private static final int BYTES_IN_MB = 1024 * 1024;
+ private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
+
+ private final List<String> affectedIndexes = new ArrayList<String>();
+ private ObjectMapper mapper = new StreamsJacksonMapper();
private ElasticsearchClientManager manager;
+ private ElasticsearchWriterConfiguration config;
private Client client;
private String parentID = null;
private BulkRequestBuilder bulkRequest;
private OutputStreamWriter currentWriter = null;
-
private int batchSize = 50;
private int totalRecordsWritten = 0;
private boolean veryLargeBulk = false; // by default this setting is set to false
- private final static Long DEFAULT_BULK_FLUSH_THRESHOLD = 5l * 1024l * 1024l;
- private static final long WAITING_DOCS_LIMIT = 10000;
+ protected Thread task;
- public volatile long flushThresholdSizeInBytes = DEFAULT_BULK_FLUSH_THRESHOLD;
+ protected volatile Queue<StreamsDatum> persistQueue;
private volatile int totalSent = 0;
private volatile int totalSeconds = 0;
@@ -67,9 +74,19 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
private volatile int totalFailed = 0;
private volatile int totalBatchCount = 0;
private volatile long totalSizeInBytes = 0;
-
private volatile long batchSizeInBytes = 0;
private volatile int batchItemsSent = 0;
+ private volatile int totalByteCount = 0;
+ private volatile int byteCount = 0;
+
+ public ElasticsearchPersistWriter() {
+ Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+ this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+ }
+
+ public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
+ this.config = config;
+ }
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
@@ -79,45 +96,58 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
this.veryLargeBulk = veryLargeBulk;
}
- private final List<String> affectedIndexes = new ArrayList<String>();
- public int getTotalOutstanding() { return this.totalSent - (this.totalFailed + this.totalOk); }
- public long getFlushThresholdSizeInBytes() { return flushThresholdSizeInBytes; }
- public int getTotalSent() { return totalSent; }
- public int getTotalSeconds() { return totalSeconds; }
- public int getTotalOk() { return totalOk; }
- public int getTotalFailed() { return totalFailed; }
- public int getTotalBatchCount() { return totalBatchCount; }
- public long getTotalSizeInBytes() { return totalSizeInBytes; }
- public long getBatchSizeInBytes() { return batchSizeInBytes; }
- public int getBatchItemsSent() { return batchItemsSent; }
- public List<String> getAffectedIndexes() { return this.affectedIndexes; }
+ public int getTotalOutstanding() {
+ return this.totalSent - (this.totalFailed + this.totalOk);
+ }
- public void setFlushThresholdSizeInBytes(long sizeInBytes) { this.flushThresholdSizeInBytes = sizeInBytes; }
+ public long getFlushThresholdSizeInBytes() {
+ return flushThresholdSizeInBytes;
+ }
- Thread task;
+ public int getTotalSent() {
+ return totalSent;
+ }
- protected volatile Queue<StreamsDatum> persistQueue;
+ public int getTotalSeconds() {
+ return totalSeconds;
+ }
- private ObjectMapper mapper = new StreamsJacksonMapper();
+ public int getTotalOk() {
+ return totalOk;
+ }
- private ElasticsearchWriterConfiguration config;
+ public int getTotalFailed() {
+ return totalFailed;
+ }
- public ElasticsearchPersistWriter() {
- Config config = StreamsConfigurator.config.getConfig("elasticsearch");
- this.config = (ElasticsearchWriterConfiguration) ElasticsearchConfigurator.detectConfiguration(config);
+ public int getTotalBatchCount() {
+ return totalBatchCount;
}
- public ElasticsearchPersistWriter(ElasticsearchWriterConfiguration config) {
- this.config = config;
+ public long getTotalSizeInBytes() {
+ return totalSizeInBytes;
}
- private static final int BYTES_IN_MB = 1024*1024;
- private static final int BYTES_BEFORE_FLUSH = 5 * BYTES_IN_MB;
- private volatile int totalByteCount = 0;
- private volatile int byteCount = 0;
+ public long getBatchSizeInBytes() {
+ return batchSizeInBytes;
+ }
- public boolean isConnected() { return (client != null); }
+ public int getBatchItemsSent() {
+ return batchItemsSent;
+ }
+
+ public List<String> getAffectedIndexes() {
+ return this.affectedIndexes;
+ }
+
+ public void setFlushThresholdSizeInBytes(long sizeInBytes) {
+ this.flushThresholdSizeInBytes = sizeInBytes;
+ }
+
+ public boolean isConnected() {
+ return (client != null);
+ }
@Override
public void write(StreamsDatum streamsDatum) {
@@ -125,7 +155,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
String json;
try {
String id = streamsDatum.getId();
- if( streamsDatum.getDocument() instanceof String )
+ if (streamsDatum.getDocument() instanceof String)
json = streamsDatum.getDocument().toString();
else {
json = mapper.writeValueAsString(streamsDatum.getDocument());
@@ -139,14 +169,6 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
}
- public void start() {
-
- manager = new ElasticsearchClientManager(config);
- client = manager.getClient();
-
- LOGGER.info(client.toString());
- }
-
public void cleanUp() {
try {
@@ -158,30 +180,25 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
@Override
- public void close()
- {
- try
- {
+ public void close() {
+ try {
// before they close, check to ensure that
this.flush();
int count = 0;
// We are going to give it 5 minutes.
- while(this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
+ while (this.getTotalOutstanding() > 0 && count++ < 20 * 60 * 5)
Thread.sleep(50);
- if(this.getTotalOutstanding() > 0)
- {
+ if (this.getTotalOutstanding() > 0) {
LOGGER.error("We never cleared our buffer");
}
- for(String indexName : this.getAffectedIndexes())
- {
+ for (String indexName : this.getAffectedIndexes()) {
createIndexIfMissing(indexName);
- if(this.veryLargeBulk)
- {
+ if (this.veryLargeBulk) {
LOGGER.debug("Resetting our Refresh Interval: {}", indexName);
// They are in 'very large bulk' mode and the process is finished. We now want to turn the
// refreshing back on.
@@ -209,9 +226,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
LOGGER.info("Closed: Wrote[{} of {}] Failed[{}]", this.getTotalOk(), this.getTotalSent(), this.getTotalFailed());
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
// this line of code should be logically unreachable.
LOGGER.warn("This is unexpected: {}", e.getMessage());
e.printStackTrace();
@@ -219,17 +234,37 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
}
@Override
- public void flush() throws IOException
- {
+ public void flush() throws IOException {
flushInternal();
}
- public void flushInternal()
- {
- synchronized (this)
- {
+ @Override
+ public void prepare(Object configurationObject) {
+ mapper = StreamsJacksonMapper.getInstance();
+ start();
+ }
+
+ @Override
+ public DatumStatusCounter getDatumStatusCounter() {
+ DatumStatusCounter counters = new DatumStatusCounter();
+ counters.incrementAttempt(this.batchItemsSent);
+ counters.incrementStatus(DatumStatus.SUCCESS, this.totalOk);
+ counters.incrementStatus(DatumStatus.FAIL, this.totalFailed);
+ return counters;
+ }
+
+ public void start() {
+
+ manager = new ElasticsearchClientManager(config);
+ client = manager.getClient();
+
+ LOGGER.info(client.toString());
+ }
+
+ public void flushInternal() {
+ synchronized (this) {
// we do not have a working bulk request, we can just exit here.
- if(this.bulkRequest == null || batchItemsSent == 0)
+ if (this.bulkRequest == null || batchItemsSent == 0)
return;
// call the flush command.
@@ -246,11 +281,9 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
this.batchSizeInBytes = 0;
this.batchItemsSent = 0;
- try
- {
+ try {
int count = 0;
- if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
- {
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT) {
/****************************************************************************
* Author:
* Smashew
@@ -275,78 +308,27 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
****************************************************************************/
// wait for the flush to catch up. We are going to cap this at
- while(this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
+ while (this.getTotalOutstanding() > WAITING_DOCS_LIMIT && count++ < 500)
Thread.sleep(10);
- if(this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
+ if (this.getTotalOutstanding() > WAITING_DOCS_LIMIT)
LOGGER.warn("Even after back-off there are {} items still in queue.", this.getTotalOutstanding());
}
- }
- catch(Exception e)
- {
+ } catch (Exception e) {
LOGGER.info("We were broken from our loop: {}", e.getMessage());
}
}
}
- private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes)
- {
- bulkRequest.execute().addListener(new ActionListener<BulkResponse>()
- {
- @Override
- public void onResponse(BulkResponse bulkItemResponses)
- {
- if (bulkItemResponses.hasFailures())
- LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
-
- long thisFailed = 0;
- long thisOk = 0;
- long thisMillis = bulkItemResponses.getTookInMillis();
-
- // keep track of the number of totalFailed and items that we have totalOk.
- for(BulkItemResponse resp : bulkItemResponses.getItems())
- {
- if(resp.isFailed())
- thisFailed++;
- else
- thisOk++;
- }
-
- totalAttempted += thisSent;
- totalOk += thisOk;
- totalFailed += thisFailed;
- totalSeconds += (thisMillis / 1000);
-
- if(thisSent != (thisOk + thisFailed))
- LOGGER.error("We sent more items than this");
-
- LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
- MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
- MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double)(1024*1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
- }
-
- @Override
- public void onFailure(Throwable e)
- {
- LOGGER.error("Error bulk loading: {}", e.getMessage());
- e.printStackTrace();
- }
- });
-
- this.notify();
- }
-
- public void add(String indexName, String type, String json)
- {
+ public void add(String indexName, String type, String json) {
add(indexName, type, null, json);
}
- public void add(String indexName, String type, String id, String json)
- {
+ public void add(String indexName, String type, String id, String json) {
IndexRequest indexRequest;
// They didn't specify an ID, so we will create one for them.
- if(id == null)
+ if (id == null)
indexRequest = new IndexRequest(indexName, type);
else
indexRequest = new IndexRequest(indexName, type, id);
@@ -356,16 +338,12 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
// If there is a parentID that is associated with this bulk, then we are
// going to have to parse the raw JSON and attempt to dereference
// what the parent document should be
- if(parentID != null)
- {
- try
- {
+ if (parentID != null) {
+ try {
// The JSONObject constructor can throw an exception, it is called
// out explicitly here so we can catch it.
indexRequest = indexRequest.parent(new JSONObject(json).getString(parentID));
- }
- catch(JSONException e)
- {
+ } catch (JSONException e) {
LOGGER.warn("Malformed JSON, cannot grab parentID: {}@{}[{}]: {}", id, indexName, type, e.getMessage());
totalFailed++;
}
@@ -373,11 +351,9 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
add(indexRequest);
}
- public void add(UpdateRequest updateRequest)
- {
+ public void add(UpdateRequest updateRequest) {
Preconditions.checkNotNull(updateRequest);
- synchronized (this)
- {
+ synchronized (this) {
checkAndCreateBulkRequest();
checkIndexImplications(updateRequest.index());
bulkRequest.add(updateRequest);
@@ -386,114 +362,57 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
Optional.fromNullable(updateRequest.doc().source().length()),
Optional.fromNullable(updateRequest.script().length()));
trackItemAndBytesWritten(size.get().longValue());
- } catch( NullPointerException x) {
+ } catch (NullPointerException x) {
trackItemAndBytesWritten(1000);
}
}
}
- public void add(IndexRequest indexRequest)
- {
- synchronized (this)
- {
+ public void add(IndexRequest indexRequest) {
+ synchronized (this) {
checkAndCreateBulkRequest();
checkIndexImplications(indexRequest.index());
bulkRequest.add(indexRequest);
try {
trackItemAndBytesWritten(indexRequest.source().length());
- } catch( NullPointerException x) {
+ } catch (NullPointerException x) {
LOGGER.warn("NPE adding/sizing indexrequest");
}
}
}
- private void trackItemAndBytesWritten(long sizeInBytes)
- {
- batchItemsSent++;
- batchSizeInBytes += sizeInBytes;
-
- // If our queue is larger than our flush threashold, then we should flush the queue.
- if(batchSizeInBytes > flushThresholdSizeInBytes)
- flushInternal();
- }
-
- private void checkAndCreateBulkRequest()
- {
- // Synchronize to ensure that we don't lose any records
- synchronized (this)
- {
- if(bulkRequest == null)
- bulkRequest = this.manager.getClient().prepareBulk();
- }
- }
-
- private void checkIndexImplications(String indexName)
- {
-
- // check to see if we have seen this index before.
- if(this.affectedIndexes.contains(indexName))
- return;
-
- // we haven't log this index.
- this.affectedIndexes.add(indexName);
-
- // Check to see if we are in 'veryLargeBulk' mode
- // if we aren't, exit early
- if(!this.veryLargeBulk)
- return;
-
-
- // They are in 'very large bulk' mode we want to turn off refreshing the index.
-
- // Create a request then add the setting to tell it to stop refreshing the interval
- UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
- updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
-
- // submit to ElasticSearch
- this.manager.getClient()
- .admin()
- .indices()
- .updateSettings(updateSettingsRequest)
- .actionGet();
- }
-
public void createIndexIfMissing(String indexName) {
- if(!this.manager.getClient()
+ if (!this.manager.getClient()
.admin()
.indices()
.exists(new IndicesExistsRequest(indexName))
.actionGet()
- .isExists())
- {
+ .isExists()) {
// It does not exist... So we are going to need to create the index.
// we are going to assume that the 'templates' that we have loaded into
// elasticsearch are sufficient to ensure the index is being created properly.
CreateIndexResponse response = this.manager.getClient().admin().indices().create(new CreateIndexRequest(indexName)).actionGet();
- if(response.isAcknowledged())
- {
+ if (response.isAcknowledged()) {
LOGGER.info("Index {} did not exist. The index was automatically created from the stored ElasticSearch Templates.", indexName);
- }
- else
- {
+ } else {
LOGGER.error("Index {} did not exist. While attempting to create the index from stored ElasticSearch Templates we were unable to get an acknowledgement.", indexName);
LOGGER.error("Error Message: {}", response.toString());
throw new RuntimeException("Unable to create index " + indexName);
}
}
}
- public void add(String indexName, String type, Map<String, Object> toImport)
- {
+
+ public void add(String indexName, String type, Map<String, Object> toImport) {
for (String id : toImport.keySet())
- add(indexName, type, id, (String)toImport.get(id));
+ add(indexName, type, id, (String) toImport.get(id));
}
- private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch)
- {
+ private void checkThenAddBatch(String index, String type, Map<String, String> workingBatch) {
Set<String> invalidIDs = checkIds(workingBatch.keySet(), index, type);
- for(String toAddId : workingBatch.keySet())
- if(!invalidIDs.contains(toAddId))
+ for (String toAddId : workingBatch.keySet())
+ if (!invalidIDs.contains(toAddId))
add(index, type, toAddId, workingBatch.get(toAddId));
LOGGER.info("Adding Batch: {} -> {}", workingBatch.size(), invalidIDs.size());
@@ -504,7 +423,7 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
IdsQueryBuilder idsFilterBuilder = new IdsQueryBuilder();
- for(String s : input)
+ for (String s : input)
idsFilterBuilder.addIds(s);
SearchRequestBuilder searchRequestBuilder = this.manager.getClient()
@@ -520,25 +439,98 @@ public class ElasticsearchPersistWriter implements StreamsPersistWriter, Flushab
Set<String> toReturn = new HashSet<String>();
- for(SearchHit hit : hits) {
+ for (SearchHit hit : hits) {
toReturn.add(hit.getId());
}
return toReturn;
}
- @Override
- public void prepare(Object configurationObject) {
- mapper = StreamsJacksonMapper.getInstance();
- start();
+ private void flush(final BulkRequestBuilder bulkRequest, final Integer thisSent, final Long thisSizeInBytes) {
+ bulkRequest.execute().addListener(new ActionListener<BulkResponse>() {
+ @Override
+ public void onResponse(BulkResponse bulkItemResponses) {
+ if (bulkItemResponses.hasFailures())
+ LOGGER.warn("Bulk Uploading had totalFailed: " + bulkItemResponses.buildFailureMessage());
+
+ long thisFailed = 0;
+ long thisOk = 0;
+ long thisMillis = bulkItemResponses.getTookInMillis();
+
+ // keep track of the number of totalFailed and items that we have totalOk.
+ for (BulkItemResponse resp : bulkItemResponses.getItems()) {
+ if (resp.isFailed())
+ thisFailed++;
+ else
+ thisOk++;
+ }
+
+ totalAttempted += thisSent;
+ totalOk += thisOk;
+ totalFailed += thisFailed;
+ totalSeconds += (thisMillis / 1000);
+
+ if (thisSent != (thisOk + thisFailed))
+ LOGGER.error("We sent more items than this");
+
+ LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]",
+ MEGABYTE_FORMAT.format((double) thisSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(thisOk), NUMBER_FORMAT.format(thisFailed), NUMBER_FORMAT.format(thisMillis),
+ MEGABYTE_FORMAT.format((double) totalSizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding()));
+ }
+
+ @Override
+ public void onFailure(Throwable e) {
+ LOGGER.error("Error bulk loading: {}", e.getMessage());
+ e.printStackTrace();
+ }
+ });
+
+ this.notify();
}
- @Override
- public DatumStatusCounter getDatumStatusCounter() {
- DatumStatusCounter counters = new DatumStatusCounter();
- counters.incrementAttempt(this.batchItemsSent);
- counters.incrementStatus(DatumStatus.SUCCESS, this.totalOk);
- counters.incrementStatus(DatumStatus.FAIL, this.totalFailed);
- return counters;
+ private void trackItemAndBytesWritten(long sizeInBytes) {
+ batchItemsSent++;
+ batchSizeInBytes += sizeInBytes;
+
+ // If our queue is larger than our flush threashold, then we should flush the queue.
+ if (batchSizeInBytes > flushThresholdSizeInBytes)
+ flushInternal();
+ }
+
+ private void checkAndCreateBulkRequest() {
+ // Synchronize to ensure that we don't lose any records
+ synchronized (this) {
+ if (bulkRequest == null)
+ bulkRequest = this.manager.getClient().prepareBulk();
+ }
+ }
+
+ private void checkIndexImplications(String indexName) {
+
+ // check to see if we have seen this index before.
+ if (this.affectedIndexes.contains(indexName))
+ return;
+
+ // we haven't log this index.
+ this.affectedIndexes.add(indexName);
+
+ // Check to see if we are in 'veryLargeBulk' mode
+ // if we aren't, exit early
+ if (!this.veryLargeBulk)
+ return;
+
+
+ // They are in 'very large bulk' mode we want to turn off refreshing the index.
+
+ // Create a request then add the setting to tell it to stop refreshing the interval
+ UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indexName);
+ updateSettingsRequest.settings(ImmutableSettings.settingsBuilder().put("refresh_interval", -1));
+
+ // submit to ElasticSearch
+ this.manager.getClient()
+ .admin()
+ .indices()
+ .updateSettings(updateSettingsRequest)
+ .actionGet();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/af1efe8d/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
index 72df97c..c10c5d5 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/PercolateProcessor.java
@@ -30,8 +30,7 @@ import java.util.concurrent.LinkedBlockingQueue;
* [t.co behavior] https://dev.twitter.com/docs/tco-redirection-behavior
*/
-public class PercolateProcessor implements StreamsProcessor, Runnable
-{
+public class PercolateProcessor implements StreamsProcessor, Runnable {
private final static Logger LOGGER = LoggerFactory.getLogger(PercolateProcessor.class);
private ObjectMapper mapper = new ObjectMapper();
@@ -85,7 +84,7 @@ public class PercolateProcessor implements StreamsProcessor, Runnable
String json;
ObjectNode node;
// first check for valid json
- if( entry.getDocument() instanceof String ) {
+ if (entry.getDocument() instanceof String) {
json = (String) entry.getDocument();
try {
node = (ObjectNode) mapper.readTree(json);
@@ -102,7 +101,7 @@ public class PercolateProcessor implements StreamsProcessor, Runnable
ArrayNode tagArray = JsonNodeFactory.instance.arrayNode();
- for( PercolateResponse.Match match : response.getMatches()) {
+ for (PercolateResponse.Match match : response.getMatches()) {
tagArray.add(match.getId().string());
}
@@ -131,14 +130,14 @@ public class PercolateProcessor implements StreamsProcessor, Runnable
@Override
public void run() {
- while(true) {
+ while (true) {
StreamsDatum item;
try {
item = inQueue.poll();
Thread.sleep(new Random().nextInt(100));
- for( StreamsDatum entry : process(item)) {
+ for (StreamsDatum entry : process(item)) {
outQueue.offer(entry);
}
[2/3] git commit: Refactored Elasticsearh components into their own
class for better cohesion
Posted by mf...@apache.org.
Refactored Elasticsearh components into their own class for better cohesion
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/b5fd7e70
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/b5fd7e70
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/b5fd7e70
Branch: refs/heads/STREAMS-58
Commit: b5fd7e70aa81515c4278cbaec788fbf3f81b8d58
Parents: af1efe8
Author: mfranklin <mf...@apache.org>
Authored: Tue Apr 22 11:34:39 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Apr 22 11:34:39 2014 -0400
----------------------------------------------------------------------
.../ElasticsearchPersistReader.java | 308 ++++---------------
.../ElasticsearchPersistReaderTask.java | 63 ----
.../elasticsearch/ElasticsearchQuery.java | 282 +++++++++++++++++
3 files changed, 339 insertions(+), 314 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b5fd7e70/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
index 72a9954..fd2a155 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
@@ -1,25 +1,17 @@
package org.apache.streams.elasticsearch;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Objects;
-import com.google.common.collect.Lists;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Queues;
-import com.typesafe.config.Config;
-import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.*;
import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.elasticsearch.action.search.SearchRequestBuilder;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.search.sort.SortBuilders;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.io.Serializable;
import java.math.BigInteger;
import java.util.*;
import java.util.concurrent.*;
@@ -32,95 +24,23 @@ import java.util.concurrent.*;
* ************************************************************************************************************
*/
-public class ElasticsearchPersistReader implements StreamsPersistReader, Iterable<SearchHit>, Iterator<SearchHit> {
+public class ElasticsearchPersistReader implements StreamsPersistReader, Serializable {
public static final String STREAMS_ID = "ElasticsearchPersistReader";
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReader.class);
- private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
- private static final Integer DEFAULT_BATCH_SIZE = 500;
- private static final String DEFAULT_SCROLL_TIMEOUT = "5m";
protected volatile Queue<StreamsDatum> persistQueue;
- private ElasticsearchClientManager elasticsearchClientManager;
- private List<String> indexes = Lists.newArrayList();
- private List<String> types = Lists.newArrayList();
- private String[] withfields;
- private String[] withoutfields;
- private DateTime startDate;
- private DateTime endDate;
- private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
- private boolean random = false;
+ private ElasticsearchQuery elasticsearchQuery;
+ private ElasticsearchReaderConfiguration config;
private int threadPoolSize = 10;
- private int batchSize = 100;
- private String scrollTimeout = null;
-
- private ObjectMapper mapper;
-
- private ElasticsearchConfiguration config;
-
private ExecutorService executor;
- private QueryBuilder queryBuilder;
- private FilterBuilder filterBuilder;
-
- // These are private to help us manage the scroll
- private SearchRequestBuilder search;
- private SearchResponse scrollResp;
- private int scrollPositionInScroll = SCROLL_POSITION_NOT_INITIALIZED;
- private SearchHit next = null;
- private long totalHits = 0;
- private long totalRead = 0;
-
public ElasticsearchPersistReader() {
- Config config = StreamsConfigurator.config.getConfig("elasticsearch");
- this.config = ElasticsearchConfigurator.detectConfiguration(config);
- }
-
- public ElasticsearchPersistReader(ElasticsearchReaderConfiguration elasticsearchConfiguration) {
- this.elasticsearchClientManager = new ElasticsearchClientManager(elasticsearchConfiguration);
- indexes.addAll(elasticsearchConfiguration.getIndexes());
- types.addAll(elasticsearchConfiguration.getTypes());
- }
-
- public long getHitCount() {
- return this.search == null ? 0 : this.totalHits;
- }
-
- public long getReadCount() {
- return this.totalRead;
- }
-
- public double getReadPercent() {
- return (double) this.getReadCount() / (double) this.getHitCount();
- }
-
- public long getRemainingCount() {
- return this.totalRead - this.totalHits;
- }
-
- public void setBatchSize(int batchSize) {
- this.batchSize = batchSize;
}
- public void setScrollTimeout(String scrollTimeout) {
- this.scrollTimeout = scrollTimeout;
- }
-
- public void setQueryBuilder(QueryBuilder queryBuilder) {
- this.queryBuilder = queryBuilder;
- }
-
- public void setFilterBuilder(FilterBuilder filterBuilder) {
- this.filterBuilder = filterBuilder;
- }
-
- public void setWithfields(String[] withfields) {
- this.withfields = withfields;
- }
-
- public void setWithoutfields(String[] withoutfields) {
- this.withoutfields = withoutfields;
+ public ElasticsearchPersistReader(ElasticsearchReaderConfiguration config) {
+ this.config = config;
}
//PersistReader methods
@@ -128,57 +48,14 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
public void startStream() {
LOGGER.debug("startStream");
executor = Executors.newSingleThreadExecutor();
- executor.submit(new ElasticsearchPersistReaderTask(this));
+ executor.submit(new ElasticsearchPersistReaderTask(this, elasticsearchQuery));
}
@Override
public void prepare(Object o) {
-
- mapper = StreamsJacksonMapper.getInstance();
- persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
-
- // If we haven't already set up the search, then set up the search.
- if (search == null) {
- search = elasticsearchClientManager.getClient()
- .prepareSearch(indexes.toArray(new String[0]))
- .setSearchType(SearchType.SCAN)
- .setSize(Objects.firstNonNull(batchSize, DEFAULT_BATCH_SIZE).intValue())
- .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT));
-
- if (this.queryBuilder != null)
- search.setQuery(this.queryBuilder);
-
- // If the types are null, then don't specify a type
- if (this.types != null && this.types.size() > 0)
- search = search.setTypes(types.toArray(new String[0]));
-
- Integer clauses = 0;
- if (this.withfields != null || this.withoutfields != null) {
- if (this.withfields != null)
- clauses += this.withfields.length;
- if (this.withoutfields != null)
- clauses += this.withoutfields.length;
- }
-
- List<FilterBuilder> filterList = buildFilterList();
-
- FilterBuilder allFilters = andFilters(filterList);
-
- if (clauses > 0) {
- // search.setPostFilter(allFilters);
- search.setPostFilter(allFilters);
- }
-
- // TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
- if (this.random)
- search = search.addSort(SortBuilders.scriptSort("random()", "number"));
- }
-
- // We don't have a scroll, we need to create a scroll
- if (scrollResp == null) {
- scrollResp = search.execute().actionGet();
- LOGGER.trace(search.toString());
- }
+ elasticsearchQuery = this.config == null ? new ElasticsearchQuery() : new ElasticsearchQuery(config);
+ elasticsearchQuery.execute(o);
+ persistQueue = constructQueue();
}
@Override
@@ -192,12 +69,12 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
StreamsResultSet current;
synchronized (ElasticsearchPersistReader.class) {
- current = new StreamsResultSet(Queues.newConcurrentLinkedQueue(persistQueue));
+ current = new StreamsResultSet(persistQueue);
current.setCounter(new DatumStatusCounter());
// current.getCounter().add(countersCurrent);
// countersTotal.add(countersCurrent);
// countersCurrent = new DatumStatusCounter();
- persistQueue.clear();
+ persistQueue = constructQueue();
}
return current;
@@ -218,69 +95,20 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
@Override
public void cleanUp() {
+ this.shutdownAndAwaitTermination(executor);
LOGGER.info("PersistReader done");
}
- //Iterable methods
- @Override
- public Iterator<SearchHit> iterator() {
- return this;
- }
-
- //Iterator methods
- @Override
- public SearchHit next() {
- return this.next;
- }
-
- @Override
- public boolean hasNext() {
- calcNext();
- return hasRecords();
- }
-
-
- public void calcNext() {
- try {
- // We have exhausted our scroll create another scroll.
- if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
- // reset the scroll position
- scrollPositionInScroll = 0;
-
- // get the next hits of the scroll
- scrollResp = elasticsearchClientManager.getClient()
- .prepareSearchScroll(scrollResp.getScrollId())
- .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT))
- .execute()
- .actionGet();
-
- this.totalHits = scrollResp.getHits().getTotalHits();
- }
-
- // If this scroll has 0 items then we set the scroll position to -1
- // letting the iterator know that we are done.
- if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
- scrollPositionInScroll = -1;
- else {
- // get the next record
- next = scrollResp.getHits().getAt(scrollPositionInScroll);
-
- // Increment our counters
- scrollPositionInScroll += 1;
- totalRead += 1;
- }
- } catch (Exception e) {
- e.printStackTrace();
- LOGGER.error("Unexpected scrolling error: {}", e.getMessage());
- scrollPositionInScroll = -1;
- next = null;
+ protected void write(StreamsDatum entry) {
+ boolean success;
+ do {
+ success = persistQueue.offer(entry);
+ Thread.yield();
}
+ while (!success);
}
- public void remove() {
- }
-
- void shutdownAndAwaitTermination(ExecutorService pool) {
+ protected void shutdownAndAwaitTermination(ExecutorService pool) {
pool.shutdown(); // Disable new tasks from being submitted
try {
// Wait a while for existing tasks to terminate
@@ -288,7 +116,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
pool.shutdownNow(); // Cancel currently executing tasks
// Wait a while for tasks to respond to being cancelled
if (!pool.awaitTermination(10, TimeUnit.SECONDS))
- System.err.println("Pool did not terminate");
+ LOGGER.error("Pool did not terminate");
}
} catch (InterruptedException ie) {
// (Re-)Cancel if current thread also interrupted
@@ -298,72 +126,50 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Iterabl
}
}
- private boolean isCompleted() {
- return totalRead >= this.limit && hasRecords();
- }
-
- private boolean hasRecords() {
- return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
- }
-
- // copied from elasticsearch
- // if we need this again we should factor it out into a utility
- private FilterBuilder andFilters(List<FilterBuilder> filters) {
- if (filters == null || filters.size() == 0)
- return null;
-
- FilterBuilder toReturn = filters.get(0);
-
- for (int i = 1; i < filters.size(); i++)
- toReturn = FilterBuilders.andFilter(toReturn, filters.get(i));
-
- return toReturn;
+ private Queue<StreamsDatum> constructQueue() {
+ return Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
}
- private FilterBuilder orFilters(List<FilterBuilder> filters) {
- if (filters == null || filters.size() == 0)
- return null;
+ public static class ElasticsearchPersistReaderTask implements Runnable {
- FilterBuilder toReturn = filters.get(0);
+ private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReaderTask.class);
- for (int i = 1; i < filters.size(); i++)
- toReturn = FilterBuilders.orFilter(toReturn, filters.get(i));
+ private ElasticsearchPersistReader reader;
+ private ElasticsearchQuery query;
+ private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
- return toReturn;
- }
-
- private List<FilterBuilder> buildFilterList() {
-
- ArrayList<FilterBuilder> filterList = Lists.newArrayList();
+ public ElasticsearchPersistReaderTask(ElasticsearchPersistReader reader, ElasticsearchQuery query) {
+ this.reader = reader;
+ this.query = query;
+ }
- // If any withfields are specified, require that field be present
- // There must a value set also for the document to be processed
- if (this.withfields != null && this.withfields.length > 0) {
- ArrayList<FilterBuilder> withFilterList = Lists.newArrayList();
- for (String withfield : this.withfields) {
- FilterBuilder withFilter = FilterBuilders.existsFilter(withfield);
- withFilterList.add(withFilter);
+ @Override
+ public void run() {
+
+ StreamsDatum item;
+ while (query.hasNext()) {
+ SearchHit hit = query.next();
+ ObjectNode jsonObject = null;
+ try {
+ jsonObject = mapper.readValue(hit.getSourceAsString(), ObjectNode.class);
+ } catch (IOException e) {
+ e.printStackTrace();
+ break;
+ }
+ item = new StreamsDatum(jsonObject, hit.getId());
+ item.getMetadata().put("id", hit.getId());
+ item.getMetadata().put("index", hit.getIndex());
+ item.getMetadata().put("type", hit.getType());
+ reader.write(item);
}
- //filterList.add(FilterBuilders.orFilter(orFilters(withFilterList)));
- filterList.add(withFilterList.get(0));
- }
- // If any withoutfields are specified, require that field not be present
- // Document will be picked up even if present, if they do not have at least one value
- // this is annoying as it majorly impacts runtime
- // might be able to change behavior using null_field
- if (this.withoutfields != null && this.withoutfields.length > 0) {
- ArrayList<FilterBuilder> withoutFilterList = Lists.newArrayList();
- for (String withoutfield : this.withoutfields) {
- FilterBuilder withoutFilter = FilterBuilders.missingFilter(withoutfield).existence(true).nullValue(false);
- withoutFilterList.add(withoutFilter);
+ try {
+ Thread.sleep(new Random().nextInt(100));
+ } catch (InterruptedException e) {
+ LOGGER.warn("Thread interrupted", e);
}
- //filterList.add(FilterBuilders.orFilter(orFilters(withoutFilterList)));
- filterList.add(withoutFilterList.get(0));
- }
- return filterList;
+ }
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b5fd7e70/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java
deleted file mode 100644
index 2d9c951..0000000
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReaderTask.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.apache.streams.elasticsearch;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import org.apache.streams.core.StreamsDatum;
-import org.apache.streams.jackson.StreamsJacksonMapper;
-import org.apache.streams.util.ComponentUtils;
-import org.elasticsearch.search.SearchHit;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.Queue;
-import java.util.Random;
-
-public class ElasticsearchPersistReaderTask implements Runnable {
-
- private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchPersistReaderTask.class);
-
- private ElasticsearchPersistReader reader;
-
- private ObjectMapper mapper = StreamsJacksonMapper.getInstance();
-
- public ElasticsearchPersistReaderTask(ElasticsearchPersistReader reader) {
- this.reader = reader;
- }
-
- @Override
- public void run() {
-
- StreamsDatum item;
- while( reader.hasNext()) {
- SearchHit hit = reader.next();
- ObjectNode jsonObject = null;
- try {
- jsonObject = mapper.readValue(hit.getSourceAsString(), ObjectNode.class);
- } catch (IOException e) {
- e.printStackTrace();
- break;
- }
- item = new StreamsDatum(jsonObject, hit.getId());
- item.getMetadata().put("id", hit.getId());
- item.getMetadata().put("index", hit.getIndex());
- item.getMetadata().put("type", hit.getType());
- write(item);
- }
- try {
- Thread.sleep(new Random().nextInt(100));
- } catch (InterruptedException e) {}
-
- }
-
- private void write( StreamsDatum entry ) {
- boolean success;
- do {
- synchronized( ElasticsearchPersistReader.class ) {
- success = reader.persistQueue.offer(entry);
- }
- Thread.yield();
- }
- while( !success );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/b5fd7e70/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
new file mode 100644
index 0000000..8c9abda
--- /dev/null
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
@@ -0,0 +1,282 @@
+package org.apache.streams.elasticsearch;
+
+import com.google.common.collect.Lists;
+import com.google.common.base.Objects;
+import com.typesafe.config.Config;
+import org.apache.streams.config.StreamsConfigurator;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.action.search.SearchResponse;
+import org.elasticsearch.action.search.SearchType;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.joda.time.DateTime;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+public class ElasticsearchQuery implements Iterable<SearchHit>, Iterator<SearchHit>, Serializable {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchQuery.class);
+ private static final int SCROLL_POSITION_NOT_INITIALIZED = -3;
+ private static final Integer DEFAULT_BATCH_SIZE = 500;
+ private static final String DEFAULT_SCROLL_TIMEOUT = "5m";
+
+ private ElasticsearchClientManager elasticsearchClientManager;
+ private ElasticsearchConfiguration config;
+ private List<String> indexes = Lists.newArrayList();
+ private List<String> types = Lists.newArrayList();
+ private String[] withfields;
+ private String[] withoutfields;
+ private DateTime startDate;
+ private DateTime endDate;
+ private int limit = 1000 * 1000 * 1000; // we are going to set the default limit very high to 1bil
+ private boolean random = false;
+ private int batchSize = 100;
+ private String scrollTimeout = null;
+ private org.elasticsearch.index.query.QueryBuilder queryBuilder;
+ private org.elasticsearch.index.query.FilterBuilder filterBuilder;// These are private to help us manage the scroll
+ private SearchRequestBuilder search;
+ private SearchResponse scrollResp;
+ private int scrollPositionInScroll = SCROLL_POSITION_NOT_INITIALIZED;
+ private SearchHit next = null;
+ private long totalHits = 0;
+ private long totalRead = 0;
+
+ public ElasticsearchQuery() {
+ Config config = StreamsConfigurator.config.getConfig("elasticsearch");
+ this.config = ElasticsearchConfigurator.detectConfiguration(config);
+ }
+
+ public ElasticsearchQuery(ElasticsearchReaderConfiguration config) {
+ this.config = config;
+ this.elasticsearchClientManager = new ElasticsearchClientManager(config);
+ this.indexes.addAll(config.getIndexes());
+ this.types.addAll(config.getTypes());
+ }
+
+ public long getHitCount() {
+ return this.search == null ? 0 : this.totalHits;
+ }
+
+ public long getReadCount() {
+ return this.totalRead;
+ }
+
+ public double getReadPercent() {
+ return (double) this.getReadCount() / (double) this.getHitCount();
+ }
+
+ public long getRemainingCount() {
+ return this.totalRead - this.totalHits;
+ }
+
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ }
+
+ public void setScrollTimeout(String scrollTimeout) {
+ this.scrollTimeout = scrollTimeout;
+ }
+
+ public void setQueryBuilder(QueryBuilder queryBuilder) {
+ this.queryBuilder = queryBuilder;
+ }
+
+ public void setFilterBuilder(FilterBuilder filterBuilder) {
+ this.filterBuilder = filterBuilder;
+ }
+
+ public void setWithfields(String[] withfields) {
+ this.withfields = withfields;
+ }
+
+ public void setWithoutfields(String[] withoutfields) {
+ this.withoutfields = withoutfields;
+ }
+
+ public void execute(Object o) {
+
+ // If we haven't already set up the search, then set up the search.
+ if (search == null) {
+ search = elasticsearchClientManager.getClient()
+ .prepareSearch(indexes.toArray(new String[0]))
+ .setSearchType(SearchType.SCAN)
+ .setSize(Objects.firstNonNull(batchSize, DEFAULT_BATCH_SIZE).intValue())
+ .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT));
+
+ if (this.queryBuilder != null)
+ search.setQuery(this.queryBuilder);
+
+ // If the types are null, then don't specify a type
+ if (this.types != null && this.types.size() > 0)
+ search = search.setTypes(types.toArray(new String[0]));
+
+ Integer clauses = 0;
+ if (this.withfields != null || this.withoutfields != null) {
+ if (this.withfields != null)
+ clauses += this.withfields.length;
+ if (this.withoutfields != null)
+ clauses += this.withoutfields.length;
+ }
+
+ List<FilterBuilder> filterList = buildFilterList();
+
+ FilterBuilder allFilters = andFilters(filterList);
+
+ if (clauses > 0) {
+ // search.setPostFilter(allFilters);
+ search.setPostFilter(allFilters);
+ }
+
+ // TODO: Replace when all clusters are upgraded past 0.90.4 so we can implement a RANDOM scroll.
+ if (this.random)
+ search = search.addSort(SortBuilders.scriptSort("random()", "number"));
+ }
+
+ // We don't have a scroll, we need to create a scroll
+ if (scrollResp == null) {
+ scrollResp = search.execute().actionGet();
+ LOGGER.trace(search.toString());
+ }
+ }
+
+ //Iterable methods
+ @Override
+ public Iterator<SearchHit> iterator() {
+ return this;
+ }
+
+ //Iterator methods
+ @Override
+ public SearchHit next() {
+ return this.next;
+ }
+
+ @Override
+ public boolean hasNext() {
+ calcNext();
+ return hasRecords();
+ }
+
+ public void calcNext() {
+ try {
+ // We have exhausted our scroll create another scroll.
+ if (scrollPositionInScroll == SCROLL_POSITION_NOT_INITIALIZED || scrollPositionInScroll >= scrollResp.getHits().getHits().length) {
+ // reset the scroll position
+ scrollPositionInScroll = 0;
+
+ // get the next hits of the scroll
+ scrollResp = elasticsearchClientManager.getClient()
+ .prepareSearchScroll(scrollResp.getScrollId())
+ .setScroll(Objects.firstNonNull(scrollTimeout, DEFAULT_SCROLL_TIMEOUT))
+ .execute()
+ .actionGet();
+
+ this.totalHits = scrollResp.getHits().getTotalHits();
+ }
+
+ // If this scroll has 0 items then we set the scroll position to -1
+ // letting the iterator know that we are done.
+ if (scrollResp.getHits().getTotalHits() == 0 || scrollResp.getHits().getHits().length == 0)
+ scrollPositionInScroll = -1;
+ else {
+ // get the next record
+ next = scrollResp.getHits().getAt(scrollPositionInScroll);
+
+ // Increment our counters
+ scrollPositionInScroll += 1;
+ totalRead += 1;
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ LOGGER.error("Unexpected scrolling error: {}", e.getMessage());
+ scrollPositionInScroll = -1;
+ next = null;
+ }
+ }
+
+ public void remove() {
+ }
+
+ protected boolean isCompleted() {
+ return totalRead >= this.limit && hasRecords();
+ }
+
+ protected boolean hasRecords() {
+ return scrollPositionInScroll != -1 && (!(this.totalRead > this.limit));
+ }
+
+ // copied from elasticsearch
+ // if we need this again we should factor it out into a utility
+ private FilterBuilder andFilters(List<FilterBuilder> filters) {
+ if (filters == null || filters.size() == 0)
+ return null;
+
+ FilterBuilder toReturn = filters.get(0);
+
+ for (int i = 1; i < filters.size(); i++)
+ toReturn = FilterBuilders.andFilter(toReturn, filters.get(i));
+
+ return toReturn;
+ }
+
+ private FilterBuilder orFilters(List<FilterBuilder> filters) {
+ if (filters == null || filters.size() == 0)
+ return null;
+
+ FilterBuilder toReturn = filters.get(0);
+
+ for (int i = 1; i < filters.size(); i++)
+ toReturn = FilterBuilders.orFilter(toReturn, filters.get(i));
+
+ return toReturn;
+ }
+
+ private List<FilterBuilder> buildFilterList() {
+
+ // If any withfields are specified, require that field be present
+ // There must a value set also for the document to be processed
+ // If any withoutfields are specified, require that field not be present
+ // Document will be picked up even if present, if they do not have at least one value
+ // this is annoying as it majorly impacts runtime
+ // might be able to change behavior using null_field
+
+ ArrayList<FilterBuilder> filterList = Lists.newArrayList();
+
+ // If any withfields are specified, require that field be present
+ // There must a value set also for the document to be processed
+ if (this.withfields != null && this.withfields.length > 0) {
+ ArrayList<FilterBuilder> withFilterList = Lists.newArrayList();
+ for (String withfield : this.withfields) {
+ FilterBuilder withFilter = FilterBuilders.existsFilter(withfield);
+ withFilterList.add(withFilter);
+ }
+ //filterList.add(FilterBuilders.orFilter(orFilters(withFilterList)));
+ filterList.add(withFilterList.get(0));
+ }
+ // If any withoutfields are specified, require that field not be present
+ // Document will be picked up even if present, if they do not have at least one value
+ // this is annoying as it majorly impacts runtime
+ // might be able to change behavior using null_field
+ if (this.withoutfields != null && this.withoutfields.length > 0) {
+ ArrayList<FilterBuilder> withoutFilterList = Lists.newArrayList();
+ for (String withoutfield : this.withoutfields) {
+ FilterBuilder withoutFilter = FilterBuilders.missingFilter(withoutfield).existence(true).nullValue(false);
+ withoutFilterList.add(withoutFilter);
+ }
+ //filterList.add(FilterBuilders.orFilter(orFilters(withoutFilterList)));
+ filterList.add(withoutFilterList.get(0));
+ }
+
+ return filterList;
+ }
+}
\ No newline at end of file
[3/3] git commit: Updated locking to ensure multiple threads can
offer to the queue but only one can swap the reference
Posted by mf...@apache.org.
Updated locking to ensure multiple threads can offer to the queue but only one can swap the reference
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/f0085694
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/f0085694
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/f0085694
Branch: refs/heads/STREAMS-58
Commit: f00856946e6cb2ee83d05e96af29730d0f88dfe9
Parents: b5fd7e7
Author: mfranklin <mf...@apache.org>
Authored: Tue Apr 22 13:08:58 2014 -0400
Committer: mfranklin <mf...@apache.org>
Committed: Tue Apr 22 13:08:58 2014 -0400
----------------------------------------------------------------------
.../ElasticsearchPersistReader.java | 20 +++++++++++++++++---
.../elasticsearch/ElasticsearchQuery.java | 2 --
2 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f0085694/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
index fd2a155..54d29d2 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchPersistReader.java
@@ -15,6 +15,8 @@ import java.io.Serializable;
import java.math.BigInteger;
import java.util.*;
import java.util.concurrent.*;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* ***********************************************************************************************************
@@ -35,6 +37,7 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Seriali
private ElasticsearchReaderConfiguration config;
private int threadPoolSize = 10;
private ExecutorService executor;
+ private ReadWriteLock lock = new ReentrantReadWriteLock();
public ElasticsearchPersistReader() {
}
@@ -68,13 +71,16 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Seriali
StreamsResultSet current;
- synchronized (ElasticsearchPersistReader.class) {
+ try {
+ lock.writeLock().lock();
current = new StreamsResultSet(persistQueue);
current.setCounter(new DatumStatusCounter());
// current.getCounter().add(countersCurrent);
// countersTotal.add(countersCurrent);
// countersCurrent = new DatumStatusCounter();
persistQueue = constructQueue();
+ } finally {
+ lock.writeLock().unlock();
}
return current;
@@ -99,11 +105,19 @@ public class ElasticsearchPersistReader implements StreamsPersistReader, Seriali
LOGGER.info("PersistReader done");
}
+ //The locking may appear to be counter intuitive but we really don't care if multiple threads offer to the queue
+ //as it is a synchronized queue. What we do care about is that we don't want to be offering to the current reference
+ //if the queue is being replaced with a new instance
protected void write(StreamsDatum entry) {
boolean success;
do {
- success = persistQueue.offer(entry);
- Thread.yield();
+ try {
+ lock.readLock().lock();
+ success = persistQueue.offer(entry);
+ Thread.yield();
+ }finally {
+ lock.readLock().unlock();
+ }
}
while (!success);
}
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/f0085694/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
index 8c9abda..2430c41 100644
--- a/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
+++ b/streams-contrib/streams-persist-elasticsearch/src/main/java/org/apache/streams/elasticsearch/ElasticsearchQuery.java
@@ -20,8 +20,6 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
public class ElasticsearchQuery implements Iterable<SearchHit>, Iterator<SearchHit>, Serializable {