You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2019/06/10 11:11:00 UTC
[james-project] branch master updated (54c31be -> 8724fcd)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git.
from 54c31be JAMES-2551 Starting James before rabbitMQ should be supported
new 698cbc4 JAMES-2733 RabbitMQ should not dequeue deleted elements
new 8724fcd JAMES-2717 Enable parrallel inMemory testing in MPT IMAP Lucene tests
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
mpt/impl/imap-mailbox/lucenesearch/pom.xml | 12 +++
.../lucenesearch/host/LuceneSearchHostSystem.java | 102 +++++++--------------
.../org/apache/james/queue/rabbitmq/Dequeuer.java | 17 +++-
.../queue/rabbitmq/RabbitMQMailQueueTest.java | 8 --
4 files changed, 60 insertions(+), 79 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 01/02: JAMES-2733 RabbitMQ should not dequeue
deleted elements
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 698cbc496d4d3e53092416e1055fde4e8479d547
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Fri May 3 09:52:26 2019 +0700
JAMES-2733 RabbitMQ should not dequeue deleted elements
We should check upon dequeue if an element had been removed or not.
---
.../java/org/apache/james/queue/rabbitmq/Dequeuer.java | 17 +++++++++++++++--
.../james/queue/rabbitmq/RabbitMQMailQueueTest.java | 8 --------
2 files changed, 15 insertions(+), 10 deletions(-)
diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/Dequeuer.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/Dequeuer.java
index d053096..a2e7a7e 100644
--- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/Dequeuer.java
+++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/Dequeuer.java
@@ -34,6 +34,7 @@ import org.apache.mailet.Mail;
import com.github.fge.lambdas.consumers.ThrowingConsumer;
import com.rabbitmq.client.Delivery;
+
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.rabbitmq.AcknowledgableDelivery;
@@ -80,8 +81,20 @@ class Dequeuer {
.filter(getResponse -> getResponse.getBody() != null);
}
- Flux<MailQueue.MailQueueItem> deQueue() {
- return flux.flatMap(this::loadItem);
+ Flux<? extends MailQueue.MailQueueItem> deQueue() {
+ return flux.flatMap(this::loadItem)
+ .flatMap(this::filterIfDeleted);
+ }
+
+ private Mono<RabbitMQMailQueueItem> filterIfDeleted(RabbitMQMailQueueItem item) {
+ return mailQueueView.isPresent(item.getMail())
+ .flatMap(isPresent -> {
+ if (isPresent) {
+ return Mono.just(item);
+ }
+ item.done(true);
+ return Mono.empty();
+ });
}
private Mono<RabbitMQMailQueueItem> loadItem(AcknowledgableDelivery response) {
diff --git a/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
index ec9e18b..3d4d271 100644
--- a/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
+++ b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
@@ -233,14 +233,6 @@ public class RabbitMQMailQueueTest implements ManageableMailQueueContract, MailQ
}
- @Disabled("JAMES-2733 Deleted elements are still dequeued")
- @Test
- @Override
- public void deletedElementsShouldNotBeDequeued() {
-
- }
-
-
private void enqueueSomeMails(Function<Integer, String> namePattern, int emailCount) {
IntStream.rangeClosed(1, emailCount)
.forEach(Throwing.intConsumer(i -> enQueue(defaultMail()
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 02/02: JAMES-2717 Enable parrallel inMemory testing
in MPT IMAP Lucene tests
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 8724fcd7bcfefb7c303282ee0c595a5cb1f2de88
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Fri Jun 7 14:39:36 2019 +0700
JAMES-2717 Enable parrallel inMemory testing in MPT IMAP Lucene tests
InMemory mostly contributes the desired isolation. Time enhancement is
achieved through the now possible parrallel test execution.
Test time went (locally) from 7:06 to 3:19
---
mpt/impl/imap-mailbox/lucenesearch/pom.xml | 12 +++
.../lucenesearch/host/LuceneSearchHostSystem.java | 102 +++++++--------------
2 files changed, 45 insertions(+), 69 deletions(-)
diff --git a/mpt/impl/imap-mailbox/lucenesearch/pom.xml b/mpt/impl/imap-mailbox/lucenesearch/pom.xml
index befdd90..16c444b 100644
--- a/mpt/impl/imap-mailbox/lucenesearch/pom.xml
+++ b/mpt/impl/imap-mailbox/lucenesearch/pom.xml
@@ -76,4 +76,16 @@
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <reuseForks>true</reuseForks>
+ <forkCount>1C</forkCount>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
index b8358af..053aa1e 100644
--- a/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
+++ b/mpt/impl/imap-mailbox/lucenesearch/src/test/java/org/apache/james/mpt/imapmailbox/lucenesearch/host/LuceneSearchHostSystem.java
@@ -19,10 +19,6 @@
package org.apache.james.mpt.imapmailbox.lucenesearch.host;
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.NotImplementedException;
import org.apache.james.core.quota.QuotaCount;
import org.apache.james.core.quota.QuotaSize;
@@ -31,7 +27,6 @@ import org.apache.james.imap.encode.main.DefaultImapEncoderFactory;
import org.apache.james.imap.main.DefaultImapDecoderFactory;
import org.apache.james.imap.processor.main.DefaultImapProcessorFactory;
import org.apache.james.mailbox.MailboxManager;
-import org.apache.james.mailbox.MailboxSession;
import org.apache.james.mailbox.SubscriptionManager;
import org.apache.james.mailbox.inmemory.InMemoryId;
import org.apache.james.mailbox.inmemory.InMemoryMailboxManager;
@@ -44,87 +39,56 @@ import org.apache.james.metrics.logger.DefaultMetricFactory;
import org.apache.james.mpt.api.ImapFeatures;
import org.apache.james.mpt.api.ImapFeatures.Feature;
import org.apache.james.mpt.host.JamesImapHostSystem;
-import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.RAMDirectory;
import com.github.fge.lambdas.Throwing;
-import com.google.common.io.Files;
public class LuceneSearchHostSystem extends JamesImapHostSystem {
- public static final String META_DATA_DIRECTORY = "target/user-meta-data";
private static final ImapFeatures SUPPORTED_FEATURES = ImapFeatures.of(Feature.NAMESPACE_SUPPORT,
Feature.MOD_SEQ_SEARCH);
-
- private File tempFile;
private InMemoryMailboxManager mailboxManager;
private LuceneMessageSearchIndex searchIndex;
@Override
public void beforeTest() throws Exception {
super.beforeTest();
- this.tempFile = Files.createTempDir();
initFields();
}
- @Override
- public void afterTest() throws Exception {
- tempFile.deleteOnExit();
-
- resetUserMetaData();
- MailboxSession session = mailboxManager.createSystemSession("test");
- mailboxManager.startProcessingRequest(session);
- mailboxManager.endProcessingRequest(session);
- mailboxManager.logout(session, false);
- }
-
- public void resetUserMetaData() throws Exception {
- File dir = new File(META_DATA_DIRECTORY);
- if (dir.exists()) {
- FileUtils.deleteDirectory(dir);
- }
- dir.mkdirs();
- }
-
private void initFields() {
- try {
- FSDirectory fsDirectory = FSDirectory.open(tempFile);
-
- InMemoryIntegrationResources resources = InMemoryIntegrationResources.builder()
- .authenticator(authenticator)
- .authorizator(authorizator)
- .inVmEventBus()
- .defaultAnnotationLimits()
- .defaultMessageParser()
- .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
- preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), fsDirectory,
- new InMemoryMessageId.Factory(),
- preInstanciationStage.getSessionProvider())))
- .noPreDeletionHooks()
- .storeQuotaManager()
- .build();
-
- mailboxManager = resources.getMailboxManager();
-
- searchIndex = (LuceneMessageSearchIndex) resources.getSearchIndex();
- searchIndex.setEnableSuffixMatch(true);
- SubscriptionManager subscriptionManager = new StoreSubscriptionManager(mailboxManager.getMapperFactory());
-
- ImapProcessor defaultImapProcessorFactory =
- DefaultImapProcessorFactory.createDefaultProcessor(
- mailboxManager,
- resources.getMailboxManager().getEventBus(),
- subscriptionManager,
- new NoQuotaManager(),
- resources.getDefaultUserQuotaRootResolver(),
- new DefaultMetricFactory());
-
- configure(new DefaultImapDecoderFactory().buildImapDecoder(),
- new DefaultImapEncoderFactory().buildImapEncoder(),
- defaultImapProcessorFactory);
-
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ InMemoryIntegrationResources resources = InMemoryIntegrationResources.builder()
+ .authenticator(authenticator)
+ .authorizator(authorizator)
+ .inVmEventBus()
+ .defaultAnnotationLimits()
+ .defaultMessageParser()
+ .listeningSearchIndex(Throwing.function(preInstanciationStage -> new LuceneMessageSearchIndex(
+ preInstanciationStage.getMapperFactory(), new InMemoryId.Factory(), new RAMDirectory(),
+ new InMemoryMessageId.Factory(),
+ preInstanciationStage.getSessionProvider())))
+ .noPreDeletionHooks()
+ .storeQuotaManager()
+ .build();
+
+ mailboxManager = resources.getMailboxManager();
+
+ searchIndex = (LuceneMessageSearchIndex) resources.getSearchIndex();
+ searchIndex.setEnableSuffixMatch(true);
+ SubscriptionManager subscriptionManager = new StoreSubscriptionManager(mailboxManager.getMapperFactory());
+
+ ImapProcessor defaultImapProcessorFactory =
+ DefaultImapProcessorFactory.createDefaultProcessor(
+ mailboxManager,
+ resources.getMailboxManager().getEventBus(),
+ subscriptionManager,
+ new NoQuotaManager(),
+ resources.getDefaultUserQuotaRootResolver(),
+ new DefaultMetricFactory());
+
+ configure(new DefaultImapDecoderFactory().buildImapDecoder(),
+ new DefaultImapEncoderFactory().buildImapEncoder(),
+ defaultImapProcessorFactory);
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org