You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/11/29 11:02:04 UTC

[1/2] james-project git commit: JAMES-2614 Disable RabbitMQMailQueueTest::concurrentEnqueueDequeueShouldNotFail

Repository: james-project
Updated Branches:
  refs/heads/master 1d5744076 -> b076dab85


JAMES-2614 Disable RabbitMQMailQueueTest::concurrentEnqueueDequeueShouldNotFail


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/b076dab8
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/b076dab8
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/b076dab8

Branch: refs/heads/master
Commit: b076dab8568d615f88e7bbe19df8b9427324a9ab
Parents: 00302ef
Author: Benoit Tellier <bt...@linagora.com>
Authored: Thu Nov 29 13:33:20 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Thu Nov 29 18:01:28 2018 +0700

----------------------------------------------------------------------
 .../apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java  | 9 +++++++++
 1 file changed, 9 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/b076dab8/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
----------------------------------------------------------------------
diff --git a/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
index 7b7630d..0db32d6 100644
--- a/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
+++ b/server/queue/queue-rabbitmq/src/test/java/org/apache/james/queue/rabbitmq/RabbitMQMailQueueTest.java
@@ -57,6 +57,7 @@ import org.apache.james.util.streams.Iterators;
 import org.apache.mailet.Mail;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.RegisterExtension;
 
@@ -217,6 +218,14 @@ public class RabbitMQMailQueueTest implements ManageableMailQueueContract, MailQ
         assertThat(initialized).isTrue();
     }
 
+    @Disabled("JAMES-2614 RabbitMQMailQueueTest::concurrentEnqueueDequeueShouldNotFail is unstable." +
+        "The related test is disabled, and need to be re-enabled after investigation and a fix.")
+    @Test
+    @Override
+    public void concurrentEnqueueDequeueShouldNotFail() {
+
+    }
+
     private void enqueueSomeMails(Function<Integer, String> namePattern, int emailCount) {
         IntStream.rangeClosed(1, emailCount)
             .forEach(Throwing.intConsumer(i -> enQueue(defaultMail()


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org


[2/2] james-project git commit: JAMES-2550 Fix MAJOR performance problem upon RabbitMQMailQueue::dequeue

Posted by bt...@apache.org.
JAMES-2550 Fix MAJOR performance problem upon RabbitMQMailQueue::dequeue

Currently using the delete method as part of dequeue **we do read all the mailqueue content**
on each, and every dequeued mails, causing major performance issue.

As a remider, we never end up browsing while dequeuing a mail from RabbitMQ.

This commit fixes this.


Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/00302ef3
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/00302ef3
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/00302ef3

Branch: refs/heads/master
Commit: 00302ef31a1aefa77b9c00246602f7a3c62fccd6
Parents: 1d57440
Author: Benoit Tellier <bt...@linagora.com>
Authored: Tue Nov 27 11:55:27 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Thu Nov 29 18:01:28 2018 +0700

----------------------------------------------------------------------
 .../queue/rabbitmq/view/api/DeleteCondition.java     |  4 ++++
 .../view/cassandra/CassandraMailQueueMailDelete.java |  6 +++++-
 .../view/cassandra/CassandraMailQueueView.java       | 15 +++++++++++++++
 3 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/james-project/blob/00302ef3/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/api/DeleteCondition.java
----------------------------------------------------------------------
diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/api/DeleteCondition.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/api/DeleteCondition.java
index 4424750..96060dd 100644
--- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/api/DeleteCondition.java
+++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/api/DeleteCondition.java
@@ -86,6 +86,10 @@ public interface DeleteCondition {
             this.name = name;
         }
 
+        public String getName() {
+            return name;
+        }
+
         @Override
         public boolean shouldBeDeleted(Mail mail) {
             Preconditions.checkNotNull(mail);

http://git-wip-us.apache.org/repos/asf/james-project/blob/00302ef3/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueMailDelete.java
----------------------------------------------------------------------
diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueMailDelete.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueMailDelete.java
index 4a9ee83..b94b1ed 100644
--- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueMailDelete.java
+++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueMailDelete.java
@@ -54,8 +54,12 @@ public class CassandraMailQueueMailDelete {
     }
 
     CompletableFuture<Void> considerDeleted(Mail mail, MailQueueName mailQueueName) {
+        return considerDeleted(MailKey.fromMail(mail), mailQueueName);
+    }
+
+    CompletableFuture<Void> considerDeleted(MailKey mailKey, MailQueueName mailQueueName) {
         return deletedMailsDao
-            .markAsDeleted(mailQueueName, MailKey.fromMail(mail))
+            .markAsDeleted(mailQueueName, mailKey)
             .thenRunAsync(() -> maybeUpdateBrowseStart(mailQueueName));
     }
 

http://git-wip-us.apache.org/repos/asf/james-project/blob/00302ef3/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueView.java
----------------------------------------------------------------------
diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueView.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueView.java
index 5f9b8f5..4980f46 100644
--- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueView.java
+++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/view/cassandra/CassandraMailQueueView.java
@@ -32,6 +32,7 @@ import org.apache.james.queue.rabbitmq.view.api.MailQueueView;
 import org.apache.james.queue.rabbitmq.view.cassandra.configuration.CassandraMailQueueViewConfiguration;
 import org.apache.james.queue.rabbitmq.view.cassandra.configuration.EventsourcingConfigurationManagement;
 import org.apache.james.queue.rabbitmq.view.cassandra.model.EnqueuedItemWithSlicingContext;
+import org.apache.james.queue.rabbitmq.view.cassandra.model.MailKey;
 import org.apache.james.util.FluentFutureStream;
 import org.apache.mailet.Mail;
 
@@ -105,6 +106,16 @@ public class CassandraMailQueueView implements MailQueueView {
 
     @Override
     public CompletableFuture<Long> delete(DeleteCondition deleteCondition) {
+        if (deleteCondition instanceof DeleteCondition.WithName) {
+            DeleteCondition.WithName nameDeleteCondition = (DeleteCondition.WithName) deleteCondition;
+
+            return delete(MailKey.of(nameDeleteCondition.getName())).thenApply(any -> 1L);
+        }
+
+        return browseThenDelete(deleteCondition);
+    }
+
+    private CompletableFuture<Long> browseThenDelete(DeleteCondition deleteCondition) {
         CompletableFuture<Long> result = cassandraMailQueueBrowser.browseReferences(mailQueueName)
             .map(EnqueuedItemWithSlicingContext::getEnqueuedItem)
             .filter(mailReference -> deleteCondition.shouldBeDeleted(mailReference.getMail()))
@@ -118,6 +129,10 @@ public class CassandraMailQueueView implements MailQueueView {
         return result;
     }
 
+    private CompletableFuture<Void> delete(MailKey mailKey) {
+        return cassandraMailQueueMailDelete.considerDeleted(mailKey, mailQueueName);
+    }
+
     @Override
     public CompletableFuture<Boolean> isPresent(Mail mail) {
         return cassandraMailQueueMailDelete.isDeleted(mail, mailQueueName)


---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org