You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2018/10/10 08:48:14 UTC
[01/14] james-project git commit: JAMES-2544 Use Feign retry
mechanism to retry calls on RabbitMQ management API
Repository: james-project
Updated Branches:
refs/heads/master eec12864d -> 34f102990
JAMES-2544 Use Feign retry mechanism to retry calls on RabbitMQ management API
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/01352919
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/01352919
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/01352919
Branch: refs/heads/master
Commit: 01352919615cb49d70934e03759e885cb69dc7ad
Parents: eec1286
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 09:12:16 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 09:15:24 2018 +0700
----------------------------------------------------------------------
.../james/queue/rabbitmq/RabbitMQManagementApi.java | 13 +++++++++++++
1 file changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/01352919/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQManagementApi.java
----------------------------------------------------------------------
diff --git a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQManagementApi.java b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQManagementApi.java
index 8d39b24..cbeea3a 100644
--- a/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQManagementApi.java
+++ b/server/queue/queue-rabbitmq/src/main/java/org/apache/james/queue/rabbitmq/RabbitMQManagementApi.java
@@ -19,6 +19,7 @@
package org.apache.james.queue.rabbitmq;
+import java.util.Date;
import java.util.List;
import java.util.stream.Stream;
@@ -30,13 +31,23 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import feign.Feign;
import feign.Logger;
import feign.RequestLine;
+import feign.RetryableException;
+import feign.Retryer;
import feign.auth.BasicAuthRequestInterceptor;
+import feign.codec.ErrorDecoder;
import feign.jackson.JacksonDecoder;
import feign.jackson.JacksonEncoder;
import feign.slf4j.Slf4jLogger;
class RabbitMQManagementApi {
+ private static final ErrorDecoder RETRY_500 = (methodKey, response) -> {
+ if (response.status() == 500) {
+ throw new RetryableException("Error encountered, scheduling retry", response.request().httpMethod(), new Date());
+ }
+ throw new RuntimeException("Non recoverable exception status: " + response.status());
+ };
+
public interface Api {
class MessageQueue {
@@ -58,6 +69,8 @@ class RabbitMQManagementApi {
.logLevel(Logger.Level.FULL)
.encoder(new JacksonEncoder())
.decoder(new JacksonDecoder())
+ .retryer(new Retryer.Default())
+ .errorDecoder(RETRY_500)
.target(Api.class, configuration.getManagementUri().toString());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[04/14] james-project git commit: JAMES-2556 Fix reprocessing when
several repositories with same path
Posted by bt...@apache.org.
JAMES-2556 Fix reprocessing when several repositories with same path
We need to aggregate repositories to check for mail existence before reprocessing
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/9d52d569
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/9d52d569
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/9d52d569
Branch: refs/heads/master
Commit: 9d52d569f749a28163719ea0ee709a55153ebbc4
Parents: f15655b
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:43:41 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:19 2018 +0700
----------------------------------------------------------------------
.../james/webadmin/service/ReprocessingService.java | 15 +++++++++------
.../webadmin/routes/MailRepositoriesRoutesTest.java | 2 --
2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/9d52d569/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
index c316b95..6ccb045 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
@@ -25,6 +25,7 @@ import java.util.function.Consumer;
import javax.inject.Inject;
import javax.mail.MessagingException;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.james.mailrepository.api.MailKey;
import org.apache.james.mailrepository.api.MailRepository;
import org.apache.james.mailrepository.api.MailRepositoryPath;
@@ -90,13 +91,15 @@ public class ReprocessingService {
public void reprocess(MailRepositoryPath path, MailKey key, Optional<String> targetProcessor, String targetQueue) throws MailRepositoryStore.MailRepositoryStoreException, MessagingException {
Reprocessor reprocessor = new Reprocessor(getMailQueue(targetQueue), targetProcessor);
- mailRepositoryStoreService
+ Pair<MailRepository, Mail> mailPair = mailRepositoryStoreService
.getRepositories(path)
- .forEach(Throwing.consumer((MailRepository repository) ->
- reprocessor.reprocess(repository,
- Optional.ofNullable(repository.retrieve(key))
- .orElseThrow(() -> new MissingKeyException(key))))
- .sneakyThrow());
+ .map(Throwing.function(repository -> Pair.of(repository, Optional.ofNullable(repository.retrieve(key)))))
+ .filter(pair -> pair.getRight().isPresent())
+ .map(pair -> Pair.of(pair.getLeft(), pair.getRight().get()))
+ .findFirst()
+ .orElseThrow(() -> new MissingKeyException(key));
+
+ reprocessor.reprocess(mailPair.getKey(), mailPair.getValue());
}
private MailQueue getMailQueue(String targetQueue) {
http://git-wip-us.apache.org/repos/asf/james-project/blob/9d52d569/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
index ea78e4c..7d98451 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
@@ -83,7 +83,6 @@ import org.apache.mailet.base.test.FakeMail;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.After;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Sets;
@@ -1506,7 +1505,6 @@ public class MailRepositoriesRoutesTest {
.body("completedDate", is(notNullValue()));
}
- @Ignore
@Test
public void reprocessingOneTaskShouldNotFailWhenSeveralRepositoryWithSamePath() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[13/14] james-project git commit: JAMES-2554 Add test on Cassandra
MailRepository storing implementation The aim of those tests is to prouve the
order of storing data: MimeMessage -> message blob parts -> keys -> mail
repository size
Posted by bt...@apache.org.
JAMES-2554 Add test on Cassandra MailRepository storing implementation
The aim of those tests is to prouve the order of storing data:
MimeMessage -> message blob parts -> keys -> mail repository size
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/e536e62f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/e536e62f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/e536e62f
Branch: refs/heads/master
Commit: e536e62fabe75a5463f345a4b72db5f8dcadc162
Parents: 33bfd0d
Author: Antoine Duprat <ad...@linagora.com>
Authored: Mon Oct 8 15:20:36 2018 +0200
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:47:38 2018 +0700
----------------------------------------------------------------------
...ilRepositoryWithFakeImplementationsTest.java | 291 +++++++++++++++++++
1 file changed, 291 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/e536e62f/server/mailrepository/mailrepository-cassandra/src/test/java/org/apache/james/mailrepository/cassandra/CassandraMailRepositoryWithFakeImplementationsTest.java
----------------------------------------------------------------------
diff --git a/server/mailrepository/mailrepository-cassandra/src/test/java/org/apache/james/mailrepository/cassandra/CassandraMailRepositoryWithFakeImplementationsTest.java b/server/mailrepository/mailrepository-cassandra/src/test/java/org/apache/james/mailrepository/cassandra/CassandraMailRepositoryWithFakeImplementationsTest.java
new file mode 100644
index 0000000..75aa16a
--- /dev/null
+++ b/server/mailrepository/mailrepository-cassandra/src/test/java/org/apache/james/mailrepository/cassandra/CassandraMailRepositoryWithFakeImplementationsTest.java
@@ -0,0 +1,291 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james.mailrepository.cassandra;
+
+import com.datastax.driver.core.ResultSet;
+import com.datastax.driver.core.Session;
+import com.google.common.collect.ImmutableList;
+import org.apache.james.backends.cassandra.CassandraCluster;
+import org.apache.james.backends.cassandra.CassandraClusterExtension;
+import org.apache.james.backends.cassandra.components.CassandraModule;
+import org.apache.james.backends.cassandra.init.CassandraTypesProvider;
+import org.apache.james.backends.cassandra.utils.CassandraUtils;
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.blob.api.Store;
+import org.apache.james.blob.cassandra.BlobTable;
+import org.apache.james.blob.cassandra.CassandraBlobModule;
+import org.apache.james.blob.cassandra.CassandraBlobsDAO;
+import org.apache.james.blob.mail.MimeMessagePartsId;
+import org.apache.james.blob.mail.MimeMessageStore;
+import org.apache.james.core.MailAddress;
+import org.apache.james.core.builder.MimeMessageBuilder;
+import org.apache.james.mailrepository.api.MailKey;
+import org.apache.james.mailrepository.api.MailRepositoryUrl;
+import org.apache.james.server.core.MailImpl;
+import org.apache.mailet.Mail;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+import javax.mail.MessagingException;
+import javax.mail.internet.MimeMessage;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+
+import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+@ExtendWith(CassandraMailRepositoryWithFakeImplementationsTest.MailRepositoryCassandraClusterExtension.class)
+class CassandraMailRepositoryWithFakeImplementationsTest {
+ static final MailRepositoryUrl URL = MailRepositoryUrl.from("proto://url");
+ static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
+
+ static class MailRepositoryCassandraClusterExtension extends CassandraClusterExtension {
+ public MailRepositoryCassandraClusterExtension() {
+ super(CassandraModule.aggregateModules(
+ CassandraMailRepositoryModule.MODULE,
+ CassandraBlobModule.MODULE));
+ }
+
+ @Override
+ public void afterAll(ExtensionContext extensionContext) {
+ }
+ }
+
+ @Nested
+ class FailingStoreTest {
+ CassandraMailRepository cassandraMailRepository;
+ CassandraMailRepositoryKeysDAO keysDAO;
+
+ @BeforeEach
+ void setup(CassandraCluster cassandra) {
+ CassandraMailRepositoryMailDAO mailDAO = new CassandraMailRepositoryMailDAO(cassandra.getConf(), BLOB_ID_FACTORY, cassandra.getTypesProvider());
+ keysDAO = new CassandraMailRepositoryKeysDAO(cassandra.getConf(), CassandraUtils.WITH_DEFAULT_CONFIGURATION);
+ CassandraMailRepositoryCountDAO countDAO = new CassandraMailRepositoryCountDAO(cassandra.getConf());
+ CassandraBlobsDAO blobsDAO = new CassandraBlobsDAO(cassandra.getConf());
+
+ cassandraMailRepository = new CassandraMailRepository(URL,
+ keysDAO, countDAO, mailDAO, new FailingStore());
+ }
+
+ class FailingStore implements Store<MimeMessage, MimeMessagePartsId> {
+
+ @Override
+ public CompletableFuture<MimeMessagePartsId> save(MimeMessage mimeMessage) {
+ return CompletableFuture.supplyAsync(() -> {
+ throw new RuntimeException("Expected failure while saving");
+ });
+ }
+
+ @Override
+ public CompletableFuture<MimeMessage> read(MimeMessagePartsId blobIds) {
+ return CompletableFuture.supplyAsync(() -> {
+ throw new RuntimeException("Expected failure while reading");
+ });
+ }
+ }
+
+ @Test
+ void keysShouldNotBeStoredWhenStoringMimeMessageHasFailed() throws Exception {
+ MailKey mailKey = new MailKey("mymail");
+ List<MailAddress> recipients = ImmutableList
+ .of(new MailAddress("rec1@domain.com"),
+ new MailAddress("rec2@domain.com"));
+ MimeMessage mailContent = MimeMessageBuilder.mimeMessageBuilder()
+ .setSubject("test")
+ .setText("this is the content")
+ .build();
+ MailImpl mail = new MailImpl(mailKey.asString(), new MailAddress("sender@domain.com"), recipients, mailContent);
+
+ assertThatThrownBy(() -> cassandraMailRepository.store(mail))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("java.lang.RuntimeException: Expected failure while saving");
+
+ assertThat(keysDAO.list(URL).join()).isEmpty();
+ }
+ }
+
+ @Nested
+ class FailingMailDaoTest {
+ CassandraMailRepository cassandraMailRepository;
+ CassandraMailRepositoryKeysDAO keysDAO;
+
+ @BeforeEach
+ void setup(CassandraCluster cassandra) {
+ FailingMailDAO mailDAO = new FailingMailDAO(cassandra.getConf(), BLOB_ID_FACTORY, cassandra.getTypesProvider());
+ keysDAO = new CassandraMailRepositoryKeysDAO(cassandra.getConf(), CassandraUtils.WITH_DEFAULT_CONFIGURATION);
+ CassandraMailRepositoryCountDAO countDAO = new CassandraMailRepositoryCountDAO(cassandra.getConf());
+ CassandraBlobsDAO blobsDAO = new CassandraBlobsDAO(cassandra.getConf());
+
+ cassandraMailRepository = new CassandraMailRepository(URL,
+ keysDAO, countDAO, mailDAO, MimeMessageStore.factory(blobsDAO).mimeMessageStore());
+ }
+
+ class FailingMailDAO extends CassandraMailRepositoryMailDAO {
+
+ public FailingMailDAO(Session session, BlobId.Factory blobIdFactory, CassandraTypesProvider cassandraTypesProvider) {
+ super(session, blobIdFactory, cassandraTypesProvider);
+ }
+
+ @Override
+ public CompletableFuture<Void> store(MailRepositoryUrl url, Mail mail, BlobId headerId, BlobId bodyId) throws MessagingException {
+ return CompletableFuture.supplyAsync(() -> {
+ throw new RuntimeException("Expected failure while storing mail parts");
+ });
+ }
+ }
+
+ @Test
+ void keysShouldNotBeStoredWhenStoringMailPartsHasFailed() throws Exception {
+ MailKey mailKey = new MailKey("mymail");
+ List<MailAddress> recipients = ImmutableList
+ .of(new MailAddress("rec1@domain.com"),
+ new MailAddress("rec2@domain.com"));
+ MimeMessage mailContent = MimeMessageBuilder.mimeMessageBuilder()
+ .setSubject("test")
+ .setText("this is the content")
+ .build();
+ MailImpl mail = new MailImpl(mailKey.asString(), new MailAddress("sender@domain.com"), recipients, mailContent);
+
+ assertThatThrownBy(() -> cassandraMailRepository.store(mail))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("java.lang.RuntimeException: Expected failure while storing mail parts");
+
+ assertThat(keysDAO.list(URL).join()).isEmpty();
+ }
+
+ @Test
+ void mimeMessageShouldBeStoredWhenStoringMailPartsHasFailed(CassandraCluster cassandra) throws Exception {
+ MailKey mailKey = new MailKey("mymail");
+ List<MailAddress> recipients = ImmutableList
+ .of(new MailAddress("rec1@domain.com"),
+ new MailAddress("rec2@domain.com"));
+ MimeMessage mailContent = MimeMessageBuilder.mimeMessageBuilder()
+ .setSubject("test")
+ .setText("this is the content")
+ .build();
+ MailImpl mail = new MailImpl(mailKey.asString(), new MailAddress("sender@domain.com"), recipients, mailContent);
+
+ assertThatThrownBy(() -> cassandraMailRepository.store(mail))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("java.lang.RuntimeException: Expected failure while storing mail parts");
+
+ ResultSet resultSet = cassandra.getConf().execute(select()
+ .from(BlobTable.TABLE_NAME));
+ assertThat(resultSet.all()).hasSize(2);
+ }
+ }
+
+ @Nested
+ class FailingKeysDaoTest {
+ CassandraMailRepository cassandraMailRepository;
+ CassandraMailRepositoryCountDAO countDAO;
+
+ @BeforeEach
+ void setup(CassandraCluster cassandra) {
+ CassandraMailRepositoryMailDAO mailDAO = new CassandraMailRepositoryMailDAO(cassandra.getConf(), BLOB_ID_FACTORY, cassandra.getTypesProvider());
+ FailingKeysDAO keysDAO = new FailingKeysDAO(cassandra.getConf(), CassandraUtils.WITH_DEFAULT_CONFIGURATION);
+ countDAO = new CassandraMailRepositoryCountDAO(cassandra.getConf());
+ CassandraBlobsDAO blobsDAO = new CassandraBlobsDAO(cassandra.getConf());
+
+ cassandraMailRepository = new CassandraMailRepository(URL,
+ keysDAO, countDAO, mailDAO, MimeMessageStore.factory(blobsDAO).mimeMessageStore());
+ }
+
+ class FailingKeysDAO extends CassandraMailRepositoryKeysDAO {
+
+ public FailingKeysDAO(Session session, CassandraUtils cassandraUtils) {
+ super(session, cassandraUtils);
+ }
+
+ @Override
+ public CompletableFuture<Boolean> store(MailRepositoryUrl url, MailKey key) {
+ return CompletableFuture.supplyAsync(() -> {
+ throw new RuntimeException("Expected failure while storing keys");
+ });
+ }
+ }
+
+ @Test
+ void sizeShouldNotBeIncreasedWhenStoringKeysHasFailed() throws Exception {
+ MailKey mailKey = new MailKey("mymail");
+ List<MailAddress> recipients = ImmutableList
+ .of(new MailAddress("rec1@domain.com"),
+ new MailAddress("rec2@domain.com"));
+ MimeMessage mailContent = MimeMessageBuilder.mimeMessageBuilder()
+ .setSubject("test")
+ .setText("this is the content")
+ .build();
+ MailImpl mail = new MailImpl(mailKey.asString(), new MailAddress("sender@domain.com"), recipients, mailContent);
+
+ assertThatThrownBy(() -> cassandraMailRepository.store(mail))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("java.lang.RuntimeException: Expected failure while storing keys");
+
+ assertThat(countDAO.getCount(URL).join()).isEqualTo(0);
+ }
+
+ @Test
+ void mimeMessageShouldBeStoredWhenStoringKeysHasFailed(CassandraCluster cassandra) throws Exception {
+ MailKey mailKey = new MailKey("mymail");
+ List<MailAddress> recipients = ImmutableList
+ .of(new MailAddress("rec1@domain.com"),
+ new MailAddress("rec2@domain.com"));
+ MimeMessage mailContent = MimeMessageBuilder.mimeMessageBuilder()
+ .setSubject("test")
+ .setText("this is the content")
+ .build();
+ MailImpl mail = new MailImpl(mailKey.asString(), new MailAddress("sender@domain.com"), recipients, mailContent);
+
+ assertThatThrownBy(() -> cassandraMailRepository.store(mail))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("java.lang.RuntimeException: Expected failure while storing keys");
+
+ ResultSet resultSet = cassandra.getConf().execute(select()
+ .from(BlobTable.TABLE_NAME));
+ assertThat(resultSet.all()).hasSize(2);
+ }
+
+ @Test
+ void mailPartsShouldBeStoredWhenStoringKeysHasFailed(CassandraCluster cassandra) throws Exception {
+ MailKey mailKey = new MailKey("mymail");
+ List<MailAddress> recipients = ImmutableList
+ .of(new MailAddress("rec1@domain.com"),
+ new MailAddress("rec2@domain.com"));
+ MimeMessage mailContent = MimeMessageBuilder.mimeMessageBuilder()
+ .setSubject("test")
+ .setText("this is the content")
+ .build();
+ MailImpl mail = new MailImpl(mailKey.asString(), new MailAddress("sender@domain.com"), recipients, mailContent);
+
+ assertThatThrownBy(() -> cassandraMailRepository.store(mail))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessage("java.lang.RuntimeException: Expected failure while storing keys");
+
+ ResultSet resultSet = cassandra.getConf().execute(select()
+ .from(MailRepositoryTable.CONTENT_TABLE_NAME));
+ assertThat(resultSet.all()).hasSize(1);
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[07/14] james-project git commit: JAMES-2556 Remove unused
MailRepositoryStoreService::getRepository
Posted by bt...@apache.org.
JAMES-2556 Remove unused MailRepositoryStoreService::getRepository
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/95b4ead2
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/95b4ead2
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/95b4ead2
Branch: refs/heads/master
Commit: 95b4ead22d4fd4d934f848c10579b258ab0d4899
Parents: d68a63e
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:22:48 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:19 2018 +0700
----------------------------------------------------------------------
.../james/webadmin/service/MailRepositoryStoreService.java | 9 ---------
1 file changed, 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/95b4ead2/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
index a626655..987b443 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
@@ -105,15 +105,6 @@ public class MailRepositoryStoreService {
return new ClearMailRepositoryTask(getRepositories(path).collect(Guavate.toImmutableList()), path);
}
- public MailRepository getRepository(MailRepositoryUrl url) throws MailRepositoryStore.MailRepositoryStoreException {
- return mailRepositoryStore.get(url)
- .orElseThrow(() -> ErrorResponder.builder()
- .statusCode(HttpStatus.NOT_FOUND_404)
- .type(ErrorResponder.ErrorType.NOT_FOUND)
- .message(url.asString() + " does not exist")
- .haltError());
- }
-
public Stream<MailRepository> getRepositories(MailRepositoryPath path) throws MailRepositoryStore.MailRepositoryStoreException {
Stream<MailRepository> byPath = mailRepositoryStore.getByPath(path);
List<MailRepository> repositories = byPath.collect(Collectors.toList());
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[10/14] james-project git commit: JAMES-2551 Fasting up RabbitMQ
cluster startup
Posted by bt...@apache.org.
JAMES-2551 Fasting up RabbitMQ cluster startup
- No need to start App and await node 1, which is never stopper
- Doing a single big step run in parallel is more efficient than several little parallel steps (-30 s)
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/33bfd0d7
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/33bfd0d7
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/33bfd0d7
Branch: refs/heads/master
Commit: 33bfd0d7aa5bb30c37decee8eee6b80218300540
Parents: 86961f3
Author: Benoit Tellier <bt...@linagora.com>
Authored: Mon Oct 8 13:23:59 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:36 2018 +0700
----------------------------------------------------------------------
.../backend/rabbitmq/DockerClusterRabbitMQExtension.java | 10 ----------
.../org/apache/james/backend/rabbitmq/DockerRabbitMQ.java | 2 ++
2 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/33bfd0d7/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerClusterRabbitMQExtension.java
----------------------------------------------------------------------
diff --git a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerClusterRabbitMQExtension.java b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerClusterRabbitMQExtension.java
index 811c768..aede7c6 100644
--- a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerClusterRabbitMQExtension.java
+++ b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerClusterRabbitMQExtension.java
@@ -64,17 +64,7 @@ public class DockerClusterRabbitMQExtension implements BeforeEachCallback, After
Throwing.runnable(() -> rabbitMQ2.join(rabbitMQ1)),
Throwing.runnable(() -> rabbitMQ3.join(rabbitMQ1)));
- Runnables.runParallel(
- Throwing.runnable(rabbitMQ1::startApp),
- Throwing.runnable(rabbitMQ2::startApp),
- Throwing.runnable(rabbitMQ3::startApp));
-
cluster = new DockerRabbitMQCluster(rabbitMQ1, rabbitMQ2, rabbitMQ3);
-
- Runnables.runParallel(
- rabbitMQ1::waitForReadyness,
- rabbitMQ2::waitForReadyness,
- rabbitMQ3::waitForReadyness);
}
@Override
http://git-wip-us.apache.org/repos/asf/james-project/blob/33bfd0d7/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerRabbitMQ.java
----------------------------------------------------------------------
diff --git a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerRabbitMQ.java b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerRabbitMQ.java
index a546939..29add81 100644
--- a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerRabbitMQ.java
+++ b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/DockerRabbitMQ.java
@@ -140,6 +140,8 @@ public class DockerRabbitMQ {
public void join(DockerRabbitMQ rabbitMQ) throws Exception {
stopApp();
joinCluster(rabbitMQ);
+ startApp();
+ waitForReadyness();
}
public void stopApp() throws java.io.IOException, InterruptedException {
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[03/14] james-project git commit: JAMES-2556 Add a test proving
reprocessing failure upon concurent mail deletion
Posted by bt...@apache.org.
JAMES-2556 Add a test proving reprocessing failure upon concurent mail deletion
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/9d1d697f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/9d1d697f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/9d1d697f
Branch: refs/heads/master
Commit: 9d1d697f6a14b1b1245a846a141787c835a09271
Parents: 0d8c68a
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:32:11 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:06 2018 +0700
----------------------------------------------------------------------
.../service/ReprocessingServiceTest.java | 196 +++++++++++++++++++
1 file changed, 196 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/9d1d697f/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
new file mode 100644
index 0000000..c1d3688
--- /dev/null
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
@@ -0,0 +1,196 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.webadmin.service;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+
+import org.apache.james.filesystem.api.FileSystem;
+import org.apache.james.mailrepository.api.MailKey;
+import org.apache.james.mailrepository.api.MailRepository;
+import org.apache.james.mailrepository.api.MailRepositoryPath;
+import org.apache.james.mailrepository.api.MailRepositoryUrl;
+import org.apache.james.mailrepository.memory.MemoryMailRepositoryProvider;
+import org.apache.james.mailrepository.memory.MemoryMailRepositoryStore;
+import org.apache.james.mailrepository.memory.MemoryMailRepositoryUrlStore;
+import org.apache.james.queue.api.MailQueueFactory;
+import org.apache.james.queue.api.ManageableMailQueue;
+import org.apache.james.queue.api.RawMailQueueItemDecoratorFactory;
+import org.apache.james.queue.memory.MemoryMailQueueFactory;
+import org.apache.james.server.core.configuration.Configuration;
+import org.apache.james.server.core.configuration.FileConfigurationProvider;
+import org.apache.james.server.core.filesystem.FileSystemImpl;
+import org.apache.mailet.base.test.FakeMail;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.github.fge.lambdas.Throwing;
+import com.github.fge.lambdas.consumers.ConsumerChainer;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+public class ReprocessingServiceTest {
+ private static final String MEMORY_PROTOCOL = "memory";
+ private static final MailRepositoryPath PATH = MailRepositoryPath.from("path");
+ private static final String NAME_1 = "key-1";
+ private static final String NAME_2 = "key-2";
+ private static final String NAME_3 = "key-3";
+ private static final MailKey KEY_1 = new MailKey(NAME_1);
+ private static final MailKey KEY_2 = new MailKey(NAME_2);
+ private static final MailKey KEY_3 = new MailKey(NAME_3);
+ private static final String SPOOL = "spool";
+ private static final Consumer<MailKey> NOOP_CONSUMER = key -> {};
+ private static final Optional<String> NO_TARGET_PROCESSOR = Optional.empty();
+
+ private ReprocessingService reprocessingService;
+ private MemoryMailRepositoryStore mailRepositoryStore;
+ private MailQueueFactory<ManageableMailQueue> queueFactory;
+ private FakeMail mail1;
+ private FakeMail mail2;
+ private FakeMail mail3;
+
+ @Before
+ public void setUp() throws Exception {
+ mailRepositoryStore = createMemoryMailRepositoryStore();
+
+ queueFactory = new MemoryMailQueueFactory(new RawMailQueueItemDecoratorFactory());
+ reprocessingService = new ReprocessingService(
+ queueFactory,
+ new MailRepositoryStoreService(mailRepositoryStore));
+
+ queueFactory.createQueue(SPOOL);
+
+ mail1 = FakeMail.builder()
+ .name(NAME_1)
+ .build();
+
+ mail2 = FakeMail.builder()
+ .name(NAME_2)
+ .build();
+
+ mail3 = FakeMail.builder()
+ .name(NAME_3)
+ .build();
+ }
+
+ @Test
+ public void reprocessingOneShouldEnqueueMail() throws Exception {
+ MailRepository repository = mailRepositoryStore.select(MailRepositoryUrl.fromPathAndProtocol(PATH, MEMORY_PROTOCOL));
+ repository.store(mail1);
+ repository.store(mail2);
+ repository.store(mail3);
+
+ reprocessingService.reprocess(PATH, KEY_2, NO_TARGET_PROCESSOR, SPOOL);
+
+ assertThat(queueFactory.getQueue(SPOOL).get()
+ .browse())
+ .extracting(item -> item.getMail().getName())
+ .containsOnly(NAME_2);
+ }
+
+ @Test
+ public void reprocessingOneShouldRemoveMailFromRepository() throws Exception {
+ MailRepository repository = mailRepositoryStore.select(MailRepositoryUrl.fromPathAndProtocol(PATH, MEMORY_PROTOCOL));
+ repository.store(mail1);
+ repository.store(mail2);
+ repository.store(mail3);
+
+ reprocessingService.reprocess(PATH, KEY_2, NO_TARGET_PROCESSOR, SPOOL);
+
+ assertThat(repository.list()).containsOnly(KEY_1, KEY_3);
+ }
+
+ @Test
+ public void reprocessingShouldEmptyRepository() throws Exception {
+ MailRepository repository = mailRepositoryStore.select(MailRepositoryUrl.fromPathAndProtocol(PATH, MEMORY_PROTOCOL));
+ repository.store(mail1);
+ repository.store(mail2);
+ repository.store(mail3);
+
+ reprocessingService.reprocessAll(PATH, NO_TARGET_PROCESSOR, SPOOL, NOOP_CONSUMER);
+
+ assertThat(repository.list()).isEmpty();
+ }
+
+ @Test
+ public void reprocessingShouldEnqueueAllMails() throws Exception {
+ MailRepository repository = mailRepositoryStore.select(MailRepositoryUrl.fromPathAndProtocol(PATH, MEMORY_PROTOCOL));
+ repository.store(mail1);
+ repository.store(mail2);
+ repository.store(mail3);
+
+ reprocessingService.reprocessAll(PATH, NO_TARGET_PROCESSOR, SPOOL, NOOP_CONSUMER);
+
+ assertThat(queueFactory.getQueue(SPOOL).get()
+ .browse())
+ .extracting(item -> item.getMail().getName())
+ .containsOnly(NAME_1, NAME_2, NAME_3);
+ }
+
+ @Ignore
+ @Test
+ public void reprocessingShouldNotFailOnConcurrentDeletion() throws Exception {
+ MailRepository repository = mailRepositoryStore.select(MailRepositoryUrl.fromPathAndProtocol(PATH, MEMORY_PROTOCOL));
+ repository.store(mail1);
+ repository.store(mail2);
+ repository.store(mail3);
+
+ AtomicBoolean shouldPerformRemove = new AtomicBoolean(true);
+ ConsumerChainer<MailKey> concurrentRemoveConsumer = Throwing.consumer(key -> {
+ if (shouldPerformRemove.get()) {
+ shouldPerformRemove.set(false);
+
+ MailKey toRemove = ImmutableList.of(NAME_1, NAME_2, NAME_3)
+ .stream()
+ .map(MailKey::new)
+ .filter(candidateForRemoval -> !candidateForRemoval.equals(key))
+ .findFirst()
+ .get();
+
+ repository.remove(toRemove);
+ }
+ });
+
+ reprocessingService.reprocessAll(PATH, NO_TARGET_PROCESSOR, SPOOL, concurrentRemoveConsumer);
+
+ assertThat(queueFactory.getQueue(SPOOL).get()
+ .browse())
+ .hasSize(2);
+ }
+
+ private MemoryMailRepositoryStore createMemoryMailRepositoryStore() throws Exception {
+ Configuration configuration = Configuration.builder()
+ .workingDirectory("../")
+ .configurationFromClasspath()
+ .build();
+ FileSystem fileSystem = new FileSystemImpl(configuration.directories());
+ MemoryMailRepositoryUrlStore urlStore = new MemoryMailRepositoryUrlStore();
+ MemoryMailRepositoryStore mailRepositoryStore = new MemoryMailRepositoryStore(urlStore, Sets.newHashSet(new MemoryMailRepositoryProvider()));
+ mailRepositoryStore.configure(new FileConfigurationProvider(fileSystem, configuration)
+ .getConfiguration("mailrepositorystore"));
+ mailRepositoryStore.init();
+ return mailRepositoryStore;
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[09/14] james-project git commit: JAMES-2556 Fix limit and offset
behaviour in multi-repository context
Posted by bt...@apache.org.
JAMES-2556 Fix limit and offset behaviour in multi-repository context
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/d68a63ef
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/d68a63ef
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/d68a63ef
Branch: refs/heads/master
Commit: d68a63efda3734eb0b34bc2c7abde327762a02c5
Parents: ea70abc
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:22:00 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:19 2018 +0700
----------------------------------------------------------------------
.../service/MailRepositoryStoreService.java | 18 +++++++-----------
.../routes/MailRepositoriesRoutesTest.java | 2 --
2 files changed, 7 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/d68a63ef/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
index ff4ee76..a626655 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/MailRepositoryStoreService.java
@@ -48,8 +48,8 @@ import org.apache.mailet.Mail;
import org.eclipse.jetty.http.HttpStatus;
import com.github.fge.lambdas.Throwing;
-import com.github.fge.lambdas.functions.ThrowingFunction;
import com.github.steveash.guavate.Guavate;
+import com.google.common.collect.ImmutableList;
public class MailRepositoryStoreService {
private final MailRepositoryStore mailRepositoryStore;
@@ -70,17 +70,13 @@ public class MailRepositoryStoreService {
}
public Optional<List<MailKeyDTO>> listMails(MailRepositoryPath path, Offset offset, Limit limit) throws MailRepositoryStore.MailRepositoryStoreException, MessagingException {
- ThrowingFunction<MailRepository, Stream<MailKeyDTO>> list = repository -> list(repository, offset, limit);
- return Optional.of(getRepositories(path)
- .flatMap(Throwing.function(list).sneakyThrow())
- .collect(Guavate.toImmutableList()));
- }
+ Optional<Stream<MailKeyDTO>> maybeMails = Optional.of(getRepositories(path)
+ .flatMap(Throwing.function((MailRepository repository) -> Iterators.toStream(repository.list())).sneakyThrow())
+ .map(MailKeyDTO::new)
+ .skip(offset.getOffset()));
- private Stream<MailKeyDTO> list(MailRepository mailRepository, Offset offset, Limit limit) throws MessagingException {
- return limit.applyOnStream(
- Iterators.toStream(mailRepository.list())
- .skip(offset.getOffset()))
- .map(MailKeyDTO::new);
+ return maybeMails.map(limit::applyOnStream)
+ .map(stream -> stream.collect(ImmutableList.toImmutableList()));
}
public Optional<Long> size(MailRepositoryPath path) throws MailRepositoryStore.MailRepositoryStoreException {
http://git-wip-us.apache.org/repos/asf/james-project/blob/d68a63ef/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
index f17b20c..ea78e4c 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
@@ -329,7 +329,6 @@ public class MailRepositoriesRoutesTest {
.body("", contains("name2"));
}
- @Ignore("Limit and offset are applied on a per repository bases")
@Test
public void listingKeysShouldApplyLimitWhenSeveralRepositories() throws Exception {
MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
@@ -403,7 +402,6 @@ public class MailRepositoriesRoutesTest {
.body("", hasSize(0));
}
- @Ignore("Offset is applied on a per mail repository basis")
@Test
public void offsetShouldBeAplliedOnTheMergedViewOfMailRepositories() throws Exception {
MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[05/14] james-project git commit: JAMES-2556 Fix reprocessing upon
concurent mail deletion
Posted by bt...@apache.org.
JAMES-2556 Fix reprocessing upon concurent mail deletion
Single key reprocessing should fail on missing key while full reprocessing should not.
Because the argument list keeps growing adding a 'Reprocessor' object helps readability.
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/f15655b2
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/f15655b2
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/f15655b2
Branch: refs/heads/master
Commit: f15655b20e3e0e3b35707773ba70ddded2563403
Parents: 95b4ead
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:41:29 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:19 2018 +0700
----------------------------------------------------------------------
.../webadmin/service/ReprocessingService.java | 47 +++++++++++++++-----
.../service/ReprocessingServiceTest.java | 2 -
2 files changed, 36 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/f15655b2/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
index f02bd3f..c316b95 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/main/java/org/apache/james/webadmin/service/ReprocessingService.java
@@ -31,12 +31,38 @@ import org.apache.james.mailrepository.api.MailRepositoryPath;
import org.apache.james.mailrepository.api.MailRepositoryStore;
import org.apache.james.queue.api.MailQueue;
import org.apache.james.queue.api.MailQueueFactory;
+import org.apache.james.util.OptionalUtils;
import org.apache.james.util.streams.Iterators;
import org.apache.mailet.Mail;
import com.github.fge.lambdas.Throwing;
public class ReprocessingService {
+ public static class MissingKeyException extends RuntimeException {
+ MissingKeyException(MailKey key) {
+ super(key.asString() + " can not be found");
+ }
+ }
+
+ static class Reprocessor {
+ private final MailQueue mailQueue;
+ private final Optional<String> targetProcessor;
+
+ Reprocessor(MailQueue mailQueue, Optional<String> targetProcessor) {
+ this.mailQueue = mailQueue;
+ this.targetProcessor = targetProcessor;
+ }
+
+ private void reprocess(MailRepository repository, Mail mail) {
+ try {
+ targetProcessor.ifPresent(mail::setState);
+ mailQueue.enQueue(mail);
+ repository.remove(mail);
+ } catch (Exception e) {
+ throw new RuntimeException("Error encountered while reprocessing mail " + mail.getName(), e);
+ }
+ }
+ }
private final MailQueueFactory<?> mailQueueFactory;
private final MailRepositoryStoreService mailRepositoryStoreService;
@@ -49,29 +75,28 @@ public class ReprocessingService {
}
public void reprocessAll(MailRepositoryPath path, Optional<String> targetProcessor, String targetQueue, Consumer<MailKey> keyListener) throws MailRepositoryStore.MailRepositoryStoreException, MessagingException {
- MailQueue mailQueue = getMailQueue(targetQueue);
+ Reprocessor reprocessor = new Reprocessor(getMailQueue(targetQueue), targetProcessor);
mailRepositoryStoreService
.getRepositories(path)
.forEach(Throwing.consumer((MailRepository repository) ->
Iterators.toStream(repository.list())
.peek(keyListener)
- .forEach(Throwing.consumer(key -> reprocess(repository, mailQueue, key, targetProcessor)))).sneakyThrow());
+ .map(Throwing.function(key -> Optional.ofNullable(repository.retrieve(key))))
+ .flatMap(OptionalUtils::toStream)
+ .forEach(mail -> reprocessor.reprocess(repository, mail))));
}
public void reprocess(MailRepositoryPath path, MailKey key, Optional<String> targetProcessor, String targetQueue) throws MailRepositoryStore.MailRepositoryStoreException, MessagingException {
- MailQueue mailQueue = getMailQueue(targetQueue);
+ Reprocessor reprocessor = new Reprocessor(getMailQueue(targetQueue), targetProcessor);
mailRepositoryStoreService
.getRepositories(path)
- .forEach(Throwing.consumer((MailRepository repository) -> reprocess(repository, mailQueue, key, targetProcessor)).sneakyThrow());
- }
-
- private void reprocess(MailRepository repository, MailQueue mailQueue, MailKey key, Optional<String> targetProcessor) throws MessagingException {
- Mail mail = repository.retrieve(key);
- targetProcessor.ifPresent(mail::setState);
- mailQueue.enQueue(mail);
- repository.remove(key);
+ .forEach(Throwing.consumer((MailRepository repository) ->
+ reprocessor.reprocess(repository,
+ Optional.ofNullable(repository.retrieve(key))
+ .orElseThrow(() -> new MissingKeyException(key))))
+ .sneakyThrow());
}
private MailQueue getMailQueue(String targetQueue) {
http://git-wip-us.apache.org/repos/asf/james-project/blob/f15655b2/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
index c1d3688..9e09a8d 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/service/ReprocessingServiceTest.java
@@ -42,7 +42,6 @@ import org.apache.james.server.core.configuration.FileConfigurationProvider;
import org.apache.james.server.core.filesystem.FileSystemImpl;
import org.apache.mailet.base.test.FakeMail;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
import com.github.fge.lambdas.Throwing;
@@ -148,7 +147,6 @@ public class ReprocessingServiceTest {
.containsOnly(NAME_1, NAME_2, NAME_3);
}
- @Ignore
@Test
public void reprocessingShouldNotFailOnConcurrentDeletion() throws Exception {
MailRepository repository = mailRepositoryStore.select(MailRepositoryUrl.fromPathAndProtocol(PATH, MEMORY_PROTOCOL));
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[14/14] james-project git commit: JAMES-2554 Use logback in tests
Posted by bt...@apache.org.
JAMES-2554 Use logback in tests
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/34f10299
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/34f10299
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/34f10299
Branch: refs/heads/master
Commit: 34f10299003853fc30a87812ba693fcc0a3c6234
Parents: e536e62
Author: Antoine Duprat <ad...@linagora.com>
Authored: Tue Oct 9 09:50:47 2018 +0200
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:47:38 2018 +0700
----------------------------------------------------------------------
.../mailrepository-cassandra/pom.xml | 5 +++
.../src/test/resources/logback-test.xml | 43 ++++++++++++++++++++
2 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/34f10299/server/mailrepository/mailrepository-cassandra/pom.xml
----------------------------------------------------------------------
diff --git a/server/mailrepository/mailrepository-cassandra/pom.xml b/server/mailrepository/mailrepository-cassandra/pom.xml
index 479a90f..506fe9e 100644
--- a/server/mailrepository/mailrepository-cassandra/pom.xml
+++ b/server/mailrepository/mailrepository-cassandra/pom.xml
@@ -78,6 +78,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/james-project/blob/34f10299/server/mailrepository/mailrepository-cassandra/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/server/mailrepository/mailrepository-cassandra/src/test/resources/logback-test.xml b/server/mailrepository/mailrepository-cassandra/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..cf76022
--- /dev/null
+++ b/server/mailrepository/mailrepository-cassandra/src/test/resources/logback-test.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<configuration>
+
+ <contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
+ <resetJUL>true</resetJUL>
+ </contextListener>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} [%-5level] %logger{15} - %msg%n%rEx</pattern>
+ <immediateFlush>false</immediateFlush>
+ </encoder>
+ </appender>
+
+ <root level="WARN">
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+
+ <logger name="org.apache.james.mailrepository.cassandra" level="DEBUG" >
+ <appender-ref ref="CONSOLE" />
+ </logger>
+
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[06/14] james-project git commit: JAMES-2556 Add tests for multi-path
repository reprocessing
Posted by bt...@apache.org.
JAMES-2556 Add tests for multi-path repository reprocessing
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/6cf97c9d
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/6cf97c9d
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/6cf97c9d
Branch: refs/heads/master
Commit: 6cf97c9d5bbe32c77e4923767f9cc547a6c9e784
Parents: 9d1d697
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:08:38 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:19 2018 +0700
----------------------------------------------------------------------
.../routes/MailRepositoriesRoutesTest.java | 120 +++++++++++++++++++
.../src/test/resources/mailrepositorystore.xml | 5 +
2 files changed, 125 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/6cf97c9d/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
index 4ea84f7..7294348 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
@@ -83,6 +83,7 @@ import org.apache.mailet.base.test.FakeMail;
import org.eclipse.jetty.http.HttpStatus;
import org.junit.After;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Sets;
@@ -95,6 +96,7 @@ import io.restassured.parsing.Parser;
public class MailRepositoriesRoutesTest {
private static final MailRepositoryUrl URL_MY_REPO = MailRepositoryUrl.from("memory://myRepo");
+ private static final MailRepositoryUrl URL_MY_REPO_OTHER = MailRepositoryUrl.from("other://myRepo");
private static final MailRepositoryPath PATH_MY_REPO = MailRepositoryPath.from("myRepo");
private static final String PATH_ESCAPED_MY_REPO = "myRepo";
private static final String MY_REPO_MAILS = "myRepo/mails";
@@ -1045,6 +1047,36 @@ public class MailRepositoriesRoutesTest {
}
@Test
+ public void reprocessingAllTaskShouldNotFailWhenSeveralRepositoriesWithSamePath() throws Exception {
+ MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
+ mailRepositoryStore.create(URL_MY_REPO_OTHER);
+ String name1 = "name1";
+ String name2 = "name2";
+ mailRepository.store(FakeMail.builder()
+ .name(name1)
+ .build());
+ mailRepository.store(FakeMail.builder()
+ .name(name2)
+ .build());
+
+ String transport = "transport";
+ String taskId = with()
+ .param("action", "reprocess")
+ .param("queue", CUSTOM_QUEUE)
+ .param("processor", transport)
+ .patch(PATH_ESCAPED_MY_REPO + "/mails")
+ .jsonPath()
+ .get("taskId");
+
+ given()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is("completed"));
+ }
+
+ @Test
public void reprocessingAllTaskShouldClearMailRepository() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
String name1 = "name1";
@@ -1073,6 +1105,36 @@ public class MailRepositoriesRoutesTest {
}
@Test
+ public void reprocessingAllTaskShouldClearBothMailRepositoriesWhenSamePath() throws Exception {
+ MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
+ MailRepository mailRepository2 = mailRepositoryStore.create(URL_MY_REPO_OTHER);
+ String name1 = "name1";
+ String name2 = "name2";
+ mailRepository1.store(FakeMail.builder()
+ .name(name1)
+ .build());
+ mailRepository2.store(FakeMail.builder()
+ .name(name2)
+ .build());
+
+ String transport = "transport";
+ String taskId = with()
+ .param("action", "reprocess")
+ .param("queue", CUSTOM_QUEUE)
+ .param("processor", transport)
+ .patch(PATH_ESCAPED_MY_REPO + "/mails")
+ .jsonPath()
+ .get("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .get(taskId + "/await");
+
+ assertThat(mailRepository1.list()).isEmpty();
+ assertThat(mailRepository2.list()).isEmpty();
+ }
+
+ @Test
public void reprocessingAllTaskShouldEnqueueMailsOnDefaultQueue() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
mailRepository.store(FakeMail.builder()
@@ -1099,6 +1161,33 @@ public class MailRepositoriesRoutesTest {
}
@Test
+ public void reprocessingAllTaskShouldEnqueueMailsOfBothRepositoriesOnDefaultQueueWhenSamePath() throws Exception {
+ MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
+ MailRepository mailRepository2 = mailRepositoryStore.create(URL_MY_REPO_OTHER);
+ mailRepository1.store(FakeMail.builder()
+ .name(NAME_1)
+ .build());
+ mailRepository2.store(FakeMail.builder()
+ .name(NAME_2)
+ .build());
+
+ String taskId = with()
+ .param("action", "reprocess")
+ .patch(PATH_ESCAPED_MY_REPO + "/mails")
+ .jsonPath()
+ .get("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .get(taskId + "/await");
+
+ assertThat(spoolQueue.browse())
+ .extracting(ManageableMailQueue.MailQueueItemView::getMail)
+ .extracting(Mail::getName)
+ .containsOnly(NAME_1, NAME_2);
+ }
+
+ @Test
public void reprocessingAllTaskShouldPreserveStateWhenProcessorIsNotSpecified() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
String state1 = "state1";
@@ -1311,6 +1400,37 @@ public class MailRepositoriesRoutesTest {
.body("completedDate", is(notNullValue()));
}
+ @Ignore
+ @Test
+ public void reprocessingOneTaskShouldNotFailWhenSeveralRepositoryWithSamePath() throws Exception {
+ MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
+ mailRepositoryStore.create(URL_MY_REPO_OTHER);
+ String name1 = "name1";
+ String name2 = "name2";
+ mailRepository.store(FakeMail.builder()
+ .name(name1)
+ .build());
+ mailRepository.store(FakeMail.builder()
+ .name(name2)
+ .build());
+
+ String transport = "transport";
+ String taskId = with()
+ .param("action", "reprocess")
+ .param("queue", CUSTOM_QUEUE)
+ .param("processor", transport)
+ .patch(PATH_ESCAPED_MY_REPO + "/mails/" + NAME_1)
+ .jsonPath()
+ .get("taskId");
+
+ given()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is("completed"));
+ }
+
@Test
public void reprocessingOneTaskShouldRemoveMailFromRepository() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
http://git-wip-us.apache.org/repos/asf/james-project/blob/6cf97c9d/server/protocols/webadmin/webadmin-mailrepository/src/test/resources/mailrepositorystore.xml
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/resources/mailrepositorystore.xml b/server/protocols/webadmin/webadmin-mailrepository/src/test/resources/mailrepositorystore.xml
index ed3e003..4cf1428 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/resources/mailrepositorystore.xml
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/resources/mailrepositorystore.xml
@@ -26,5 +26,10 @@
<protocol>memory</protocol>
</protocols>
</mailrepository>
+ <mailrepository class="org.apache.james.mailrepository.memory.MemoryMailRepository">
+ <protocols>
+ <protocol>other</protocol>
+ </protocols>
+ </mailrepository>
</mailrepositories>
</mailrepositorystore>
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[08/14] james-project git commit: JAMES-2556 Add tests for multi-path
repository management
Posted by bt...@apache.org.
JAMES-2556 Add tests for multi-path repository management
This proves limit and offset to be failing
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/ea70abcd
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/ea70abcd
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/ea70abcd
Branch: refs/heads/master
Commit: ea70abcd16910b63822061f0e5858f7d8be1cbd3
Parents: 6cf97c9
Author: Benoit Tellier <bt...@linagora.com>
Authored: Wed Oct 10 10:16:52 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:19 2018 +0700
----------------------------------------------------------------------
.../routes/MailRepositoriesRoutesTest.java | 108 +++++++++++++++++++
1 file changed, 108 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/ea70abcd/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
----------------------------------------------------------------------
diff --git a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
index 7294348..f17b20c 100644
--- a/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailrepository/src/test/java/org/apache/james/webadmin/routes/MailRepositoriesRoutesTest.java
@@ -212,6 +212,20 @@ public class MailRepositoriesRoutesTest {
}
@Test
+ public void getMailRepositoriesShouldDeduplicateAccordingToPath() throws Exception {
+ mailRepositoryStore.create(URL_MY_REPO);
+ mailRepositoryStore.create(URL_MY_REPO_OTHER);
+
+ when()
+ .get()
+ .then()
+ .statusCode(HttpStatus.OK_200)
+ .body("", hasSize(1))
+ .body("[0].repository", is(PATH_MY_REPO.asString()))
+ .body("[0].path", is(PATH_ESCAPED_MY_REPO));
+ }
+
+ @Test
public void getMailRepositoriesShouldReturnTwoRepositoriesWhenTwo() throws Exception {
mailRepositoryStore.create(URL_MY_REPO);
mailRepositoryStore.create(MailRepositoryUrl.from("memory://mySecondRepo"));
@@ -271,6 +285,26 @@ public class MailRepositoriesRoutesTest {
}
@Test
+ public void listingKeysShouldMergeRepositoryContentWhenSamePath() throws Exception {
+ MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
+ MailRepository mailRepository2 = mailRepositoryStore.create(URL_MY_REPO_OTHER);
+
+ mailRepository1.store(FakeMail.builder()
+ .name("name1")
+ .build());
+ mailRepository2.store(FakeMail.builder()
+ .name("name2")
+ .build());
+
+ when()
+ .get(MY_REPO_MAILS)
+ .then()
+ .statusCode(HttpStatus.OK_200)
+ .body("", hasSize(2))
+ .body("", containsInAnyOrder("name1", "name2"));
+ }
+
+ @Test
public void listingKeysShouldApplyLimitAndOffset() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
@@ -295,6 +329,31 @@ public class MailRepositoriesRoutesTest {
.body("", contains("name2"));
}
+ @Ignore("Limit and offset are applied on a per repository bases")
+ @Test
+ public void listingKeysShouldApplyLimitWhenSeveralRepositories() throws Exception {
+ MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
+ MailRepository mailRepository2 = mailRepositoryStore.create(URL_MY_REPO_OTHER);
+
+ mailRepository1.store(FakeMail.builder()
+ .name("name1")
+ .build());
+ mailRepository1.store(FakeMail.builder()
+ .name("name2")
+ .build());
+ mailRepository2.store(FakeMail.builder()
+ .name("name3")
+ .build());
+
+ given()
+ .param("limit", "1")
+ .when()
+ .get(MY_REPO_MAILS)
+ .then()
+ .statusCode(HttpStatus.OK_200)
+ .body("", hasSize(1));
+ }
+
@Test
public void listingKeysShouldReturnErrorOnInvalidOffset() {
given()
@@ -344,6 +403,34 @@ public class MailRepositoriesRoutesTest {
.body("", hasSize(0));
}
+ @Ignore("Offset is applied on a per mail repository basis")
+ @Test
+ public void offsetShouldBeAplliedOnTheMergedViewOfMailRepositories() throws Exception {
+ MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
+ MailRepository mailRepository2 = mailRepositoryStore.create(URL_MY_REPO_OTHER);
+
+ mailRepository1.store(FakeMail.builder()
+ .name("name1")
+ .build());
+ mailRepository2.store(FakeMail.builder()
+ .name("name2")
+ .build());
+ mailRepository1.store(FakeMail.builder()
+ .name("name3")
+ .build());
+ mailRepository2.store(FakeMail.builder()
+ .name("name4")
+ .build());
+
+ given()
+ .param("offset", "2")
+ .when()
+ .get(MY_REPO_MAILS)
+ .then()
+ .statusCode(HttpStatus.OK_200)
+ .body("", hasSize(2));
+ }
+
@Test
public void listingKeysShouldReturnErrorOnInvalidLimit() {
given()
@@ -454,6 +541,27 @@ public class MailRepositoriesRoutesTest {
}
@Test
+ public void retrievingRepositorySizeShouldReturnNumberOfContainedMailsWhenSeveralRepositoryWithSamePath() throws Exception {
+ MailRepository mailRepository1 = mailRepositoryStore.create(URL_MY_REPO);
+ MailRepository mailRepository2 = mailRepositoryStore.create(URL_MY_REPO_OTHER);
+
+ mailRepository1.store(FakeMail.builder()
+ .name(NAME_1)
+ .build());
+
+ mailRepository2.store(FakeMail.builder()
+ .name(NAME_2)
+ .build());
+
+ given()
+ .get(PATH_ESCAPED_MY_REPO)
+ .then()
+ .statusCode(HttpStatus.OK_200)
+ .contentType(ContentType.JSON)
+ .body("size", equalTo(2));
+ }
+
+ @Test
public void retrievingAMailShouldDisplayItsInformation() throws Exception {
MailRepository mailRepository = mailRepositoryStore.create(URL_MY_REPO);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[11/14] james-project git commit: JAMES-2334 Disable
nodeKillingWhenConsuming
Posted by bt...@apache.org.
JAMES-2334 Disable nodeKillingWhenConsuming
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/86961f3f
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/86961f3f
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/86961f3f
Branch: refs/heads/master
Commit: 86961f3fc818ab3fdee2ddc76dcd16ff06e61dff
Parents: de7d0e0
Author: Benoit Tellier <bt...@linagora.com>
Authored: Mon Oct 8 10:32:50 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:36 2018 +0700
----------------------------------------------------------------------
.../org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/86961f3f/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
----------------------------------------------------------------------
diff --git a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
index 081f0d1..b72b45b 100644
--- a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
+++ b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
@@ -245,6 +245,9 @@ class RabbitMQClusterTest {
}
}
+ @Disabled("JAMES-2334 For some reason, we are unable to recover topology when reconnecting" +
+ "See https://github.com/rabbitmq/rabbitmq-server/issues/959" +
+ "This test have roughly 4% chance to fail")
@Test
void nodeKillingWhenConsuming(DockerRabbitMQCluster cluster) throws Exception {
resilientChannel.exchangeDeclare(EXCHANGE_NAME, DIRECT, DURABLE);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[02/14] james-project git commit: JAMES-2557 DLP should ignore null
sender
Posted by bt...@apache.org.
JAMES-2557 DLP should ignore null sender
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/0d8c68ab
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/0d8c68ab
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/0d8c68ab
Branch: refs/heads/master
Commit: 0d8c68ab7860f69eb1c376fc89ab04f0926e223c
Parents: 0135291
Author: Benoit Tellier <bt...@linagora.com>
Authored: Tue Oct 9 11:44:27 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:45:50 2018 +0700
----------------------------------------------------------------------
.../org/apache/james/transport/matchers/dlp/Dlp.java | 4 ++--
.../james/transport/matchers/dlp/DlpDomainRules.java | 4 +++-
.../apache/james/transport/matchers/dlp/DlpTest.java | 13 +++++++++++++
3 files changed, 18 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/0d8c68ab/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
index 1f65082..a82dc06 100644
--- a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
+++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/Dlp.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Optional;
import javax.inject.Inject;
-import javax.mail.MessagingException;
import org.apache.james.core.MailAddress;
import org.apache.james.dlp.api.DLPConfigurationItem;
@@ -51,7 +50,7 @@ public class Dlp extends GenericMatcher {
}
@Override
- public Collection<MailAddress> match(Mail mail) throws MessagingException {
+ public Collection<MailAddress> match(Mail mail) {
Optional<DLPConfigurationItem.Id> firstMatchingRuleId = findFirstMatchingRule(mail);
if (firstMatchingRuleId.isPresent()) {
@@ -69,6 +68,7 @@ public class Dlp extends GenericMatcher {
private Optional<DLPConfigurationItem.Id> findFirstMatchingRule(Mail mail) {
return Optional
.ofNullable(mail.getSender())
+ .filter(sender -> !sender.isNullSender())
.flatMap(sender -> matchingRule(sender, mail));
}
http://git-wip-us.apache.org/repos/asf/james-project/blob/0d8c68ab/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
index b954801..031f306 100644
--- a/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
+++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/matchers/dlp/DlpDomainRules.java
@@ -169,7 +169,9 @@ public class DlpDomainRules {
}
private Stream<String> listEnvelopSender(Mail mail) {
- return StreamUtils.ofNullables(mail.getSender()).map(MailAddress::asString);
+ return StreamUtils.ofNullables(mail.getSender())
+ .filter(sender -> !sender.isNullSender())
+ .map(MailAddress::asString);
}
private Stream<String> listFromHeaders(Mail mail) throws MessagingException {
http://git-wip-us.apache.org/repos/asf/james-project/blob/0d8c68ab/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
----------------------------------------------------------------------
diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
index 12aa394..88266d6 100644
--- a/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
+++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/matchers/dlp/DlpTest.java
@@ -34,6 +34,7 @@ import java.util.Optional;
import java.util.regex.Pattern;
import org.apache.james.core.Domain;
+import org.apache.james.core.MailAddress;
import org.apache.james.core.builder.MimeMessageBuilder;
import org.apache.james.dlp.api.DLPConfigurationItem.Id;
import org.apache.mailet.base.test.FakeMail;
@@ -103,6 +104,18 @@ class DlpTest {
}
@Test
+ void nullSenderShouldBeIgnored() throws Exception {
+ Dlp dlp = new Dlp(
+ asRulesLoaderFor(
+ JAMES_APACHE_ORG_DOMAIN,
+ DlpDomainRules.builder().recipientRule(Id.of("match all recipient"), Pattern.compile(".*")).build()));
+
+ FakeMail mail = FakeMail.builder().sender(MailAddress.nullSender()).recipient(RECIPIENT1).build();
+
+ assertThat(dlp.match(mail)).isEmpty();
+ }
+
+ @Test
void matchSenderShouldReturnRecipientsWhenFromHeaderMatches() throws Exception {
Dlp dlp = new Dlp(
asRulesLoaderFor(
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[12/14] james-project git commit: JAMES-2334 Use resilient channel
for queue declaration
Posted by bt...@apache.org.
JAMES-2334 Use resilient channel for queue declaration
Project: http://git-wip-us.apache.org/repos/asf/james-project/repo
Commit: http://git-wip-us.apache.org/repos/asf/james-project/commit/de7d0e0a
Tree: http://git-wip-us.apache.org/repos/asf/james-project/tree/de7d0e0a
Diff: http://git-wip-us.apache.org/repos/asf/james-project/diff/de7d0e0a
Branch: refs/heads/master
Commit: de7d0e0aaf711a6da6105788d2bbecacdbb2d967
Parents: 9d52d56
Author: Benoit Tellier <bt...@linagora.com>
Authored: Mon Oct 8 10:31:27 2018 +0700
Committer: Benoit Tellier <bt...@linagora.com>
Committed: Wed Oct 10 15:46:36 2018 +0700
----------------------------------------------------------------------
.../org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/james-project/blob/de7d0e0a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
----------------------------------------------------------------------
diff --git a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
index 6486a84..081f0d1 100644
--- a/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
+++ b/backends-common/rabbitmq/src/test/java/org/apache/james/backend/rabbitmq/RabbitMQClusterTest.java
@@ -247,9 +247,9 @@ class RabbitMQClusterTest {
@Test
void nodeKillingWhenConsuming(DockerRabbitMQCluster cluster) throws Exception {
- node2Channel.exchangeDeclare(EXCHANGE_NAME, DIRECT, DURABLE);
- node2Channel.queueDeclare(QUEUE, DURABLE, !EXCLUSIVE, !AUTO_DELETE, ImmutableMap.of()).getQueue();
- node2Channel.queueBind(QUEUE, EXCHANGE_NAME, ROUTING_KEY);
+ resilientChannel.exchangeDeclare(EXCHANGE_NAME, DIRECT, DURABLE);
+ resilientChannel.queueDeclare(QUEUE, DURABLE, !EXCLUSIVE, !AUTO_DELETE, ImmutableMap.of()).getQueue();
+ resilientChannel.queueBind(QUEUE, EXCHANGE_NAME, ROUTING_KEY);
int nbMessages = 10;
IntStream.range(0, nbMessages)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org