You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2016/04/06 11:21:54 UTC
[10/18] james-project git commit: MAILBOX-266 Set the size of
SearchResponses to 1 in ElasticSearch tests
MAILBOX-266 Set the size of SearchResponses to 1 in ElasticSearch tests
This allows to ensure Scrolls are well performed
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/b99d0c7f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/b99d0c7f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/b99d0c7f
Branch: refs/heads/master
Commit: b99d0c7f0164e431a4cbaa3e8e8fc19030ba3a6a
Parents: 6ba31a6
Author: Benoit Tellier <bt...@linagora.com>
Authored: Mon Mar 28 10:26:36 2016 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Apr 6 16:18:45 2016 +0700
----------------------------------------------------------------------
.../mailbox/elasticsearch/DeleteByQueryPerformer.java | 13 +++++++++++--
.../elasticsearch/search/ElasticSearchSearcher.java | 12 ++++++++++--
.../elasticsearch/ElasticSearchIndexerTest.java | 3 ++-
.../elasticsearch/ElasticSearchIntegrationTest.java | 6 ++++--
4 files changed, 27 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
index 9d266ce..895f016 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/DeleteByQueryPerformer.java
@@ -34,17 +34,26 @@ import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchHit;
+import com.google.common.annotations.VisibleForTesting;
+
public class DeleteByQueryPerformer {
- public static final int BATCH_SIZE = 100;
+ public static final int DEFAULT_BATCH_SIZE = 100;
public static final TimeValue TIMEOUT = new TimeValue(60000);
private final ClientProvider clientProvider;
private final ExecutorService executor;
+ private final int batchSize;
@Inject
public DeleteByQueryPerformer(ClientProvider clientProvider, @Named("AsyncExecutor") ExecutorService executor) {
+ this(clientProvider, executor, DEFAULT_BATCH_SIZE);
+ }
+
+ @VisibleForTesting
+ DeleteByQueryPerformer(ClientProvider clientProvider, @Named("AsyncExecutor") ExecutorService executor, int batchSize) {
this.clientProvider = clientProvider;
this.executor = executor;
+ this.batchSize = batchSize;
}
public void perform(QueryBuilder queryBuilder) {
@@ -59,7 +68,7 @@ public class DeleteByQueryPerformer {
.setScroll(TIMEOUT)
.setNoFields()
.setQuery(queryBuilder)
- .setSize(BATCH_SIZE));
+ .setSize(batchSize));
for (SearchResponse searchResponse : scrollIterable) {
deleteRetrievedIds(client, searchResponse);
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
index 96d90b5..ab4698e 100644
--- a/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
+++ b/mailbox/elasticsearch/src/main/java/org/apache/james/mailbox/elasticsearch/search/ElasticSearchSearcher.java
@@ -46,14 +46,22 @@ import org.slf4j.LoggerFactory;
public class ElasticSearchSearcher<Id extends MailboxId> {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchSearcher.class);
+ private static final TimeValue TIMEOUT = new TimeValue(60000);
+ public static final int DEFAULT_SIZE = 100;
private final ClientProvider clientProvider;
private final QueryConverter queryConverter;
+ private final int size;
@Inject
public ElasticSearchSearcher(ClientProvider clientProvider, QueryConverter queryConverter) {
+ this(clientProvider, queryConverter, DEFAULT_SIZE);
+ }
+
+ public ElasticSearchSearcher(ClientProvider clientProvider, QueryConverter queryConverter, int size) {
this.clientProvider = clientProvider;
this.queryConverter = queryConverter;
+ this.size = size;
}
public Iterator<Long> search(Mailbox<Id> mailbox, SearchQuery searchQuery) throws MailboxException {
@@ -70,9 +78,9 @@ public class ElasticSearchSearcher<Id extends MailboxId> {
.reduce(
client.prepareSearch(ElasticSearchIndexer.MAILBOX_INDEX)
.setTypes(ElasticSearchIndexer.MESSAGE_TYPE)
- .setScroll(new TimeValue(60000))
+ .setScroll(TIMEOUT)
.setQuery(queryConverter.from(searchQuery, mailbox.getMailboxId().serialize()))
- .setSize(100),
+ .setSize(size),
(searchBuilder, sort) -> searchBuilder.addSort(SortConverter.convertSort(sort)),
(partialResult1, partialResult2) -> partialResult1);
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
index 1e4ce1a..7501c57 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIndexerTest.java
@@ -41,6 +41,7 @@ import com.google.common.collect.Lists;
public class ElasticSearchIndexerTest {
+ public static final int MINIMUM_BATCH_SIZE = 1;
private TemporaryFolder temporaryFolder = new TemporaryFolder();
private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder);
@@ -55,7 +56,7 @@ public class ElasticSearchIndexerTest {
public void setup() throws IOException {
node = embeddedElasticSearch.getNode();
TestingClientProvider clientProvider = new TestingClientProvider(node);
- deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor());
+ deleteByQueryPerformer = new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor(), MINIMUM_BATCH_SIZE);
testee = new ElasticSearchIndexer(clientProvider, deleteByQueryPerformer);
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/b99d0c7f/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
index 2f5962e..9b90b72 100644
--- a/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
+++ b/mailbox/elasticsearch/src/test/java/org/apache/james/mailbox/elasticsearch/ElasticSearchIntegrationTest.java
@@ -62,6 +62,8 @@ import com.google.common.collect.Lists;
public class ElasticSearchIntegrationTest {
private static final Logger LOGGER = LoggerFactory.getLogger(ElasticSearchIntegrationTest.class);
+ public static final int BATCH_SIZE = 1;
+ public static final int SEARCH_SIZE = 1;
private TemporaryFolder temporaryFolder = new TemporaryFolder();
private EmbeddedElasticSearch embeddedElasticSearch= new EmbeddedElasticSearch(temporaryFolder);
@@ -168,8 +170,8 @@ public class ElasticSearchIntegrationTest {
);
MailboxSessionMapperFactory<InMemoryId> mapperFactory = new InMemoryMailboxSessionMapperFactory();
elasticSearchListeningMessageSearchIndex = new ElasticSearchListeningMessageSearchIndex<>(mapperFactory,
- new ElasticSearchIndexer(clientProvider, new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor())),
- new ElasticSearchSearcher<>(clientProvider, new QueryConverter(new CriterionConverter())),
+ new ElasticSearchIndexer(clientProvider, new DeleteByQueryPerformer(clientProvider, Executors.newSingleThreadExecutor(), BATCH_SIZE)),
+ new ElasticSearchSearcher<>(clientProvider, new QueryConverter(new CriterionConverter()), SEARCH_SIZE),
new MessageToElasticSearchJson(new DefaultTextExtractor(), ZoneId.of("Europe/Paris")));
storeMailboxManager = new StoreMailboxManager<>(
mapperFactory,
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org