You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by ro...@apache.org on 2019/05/13 12:38:09 UTC
[james-project] branch master updated (2a91b93 -> db3aad0)
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git.
from 2a91b93 JAMES-2717 Remove EmbeddedElasticSearch
new 9e2c0cf JAMES-2725 Remove length from Blob API This means that the S3 implementation should implement its own way to put blobs without knowing the length
new 9473aac JAMES-2725 Split object storage tests per implementation
new e483291 JAMES-2725 add retry policy for s3 upload
new 8e73a1d JAMES-2725 add retry on exception for s3 upload
new ad14321 JAMES-2725 remove BlobIdFactory parameter from AwsS3ObjectStorage.putBlob
new 712f005 JAMES-2725 Use injection for AwsS3Object storage for managing its thread pool lifecycle.
new db3aad0 Merge remote-tracking branch 'remk/JAMES-2725-2'
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/james/blob/api/BlobStore.java | 2 +-
.../apache/james/blob/api/MetricableBlobStore.java | 4 +-
.../main/java/org/apache/james/blob/api/Store.java | 28 +---
.../apache/james/blob/api/BlobStoreContract.java | 6 +-
.../james/blob/api/FixedLengthInputStreamTest.java | 63 -------
.../blob/api/MetricableBlobStoreContract.java | 4 +-
.../james/blob/cassandra/CassandraBlobsDAO.java | 2 +-
.../blob/cassandra/CassandraBlobsDAOTest.java | 2 +-
.../apache/james/blob/memory/MemoryBlobStore.java | 2 +-
server/blob/blob-objectstorage/pom.xml | 5 +
.../blob/objectstorage/ObjectStorageBlobsDAO.java | 17 +-
.../ObjectStorageBlobsDAOBuilder.java | 19 ++-
...AuthConfiguration.java => PutBlobFunction.java} | 15 +-
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 128 +++++++++++++-
.../objectstorage/ObjectStorageBlobsDAOTest.java | 2 +-
.../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 11 +-
.../apache/james/blob/union/UnionBlobStore.java | 8 +-
.../james/blob/union/UnionBlobStoreTest.java | 14 +-
.../apache/james/blob/mail/MimeMessageStore.java | 6 +-
.../ObjectStorageDependenciesModule.java | 19 ++-
.../james/CassandraRabbitMQJamesServerFixture.java | 43 +++++
.../james/CassandraRabbitMQJamesServerTest.java | 184 ---------------------
.../apache/james/MailsShouldBeWellReceived.java | 66 ++++++++
.../org/apache/james/WithCassandraBlobStore.java} | 45 ++---
.../apache/james/WithCassandraBlobStoreTest.java} | 6 +-
.../apache/james/WithDefaultAwsS3Extension.java} | 48 +++---
.../org/apache/james/WithDefaultAwsS3Test.java} | 34 ++--
.../apache/james/WithDefaultSwiftExtension.java} | 48 +++---
.../org/apache/james/WithDefaultSwiftTest.java} | 34 ++--
.../apache/james/WithEncryptedAwsS3Extension.java} | 49 +++---
.../org/apache/james/WithEncryptedAwsS3Test.java} | 34 ++--
.../apache/james/WithEncryptedSwiftExtension.java} | 49 +++---
.../org/apache/james/WithEncryptedSwiftTest.java} | 35 ++--
.../org/apache/james/JamesServerExtension.java | 5 +
.../james/webadmin/vault/routes/ExportService.java | 2 +-
35 files changed, 531 insertions(+), 508 deletions(-)
delete mode 100644 server/blob/blob-api/src/test/java/org/apache/james/blob/api/FixedLengthInputStreamTest.java
copy server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/{SpecificAuthConfiguration.java => PutBlobFunction.java} (69%)
create mode 100644 server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java
delete mode 100644 server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java
create mode 100644 server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java
copy server/{blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/FileSystemExtension.java => container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStore.java} (60%)
copy server/{blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/SpecificAuthConfiguration.java => container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreTest.java} (76%)
copy server/{blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/FileSystemExtension.java => container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Extension.java} (57%)
copy server/container/{util/src/test/java/org/apache/james/util/RunnablesTest.java => guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Test.java} (53%)
copy server/{blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/FileSystemExtension.java => container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftExtension.java} (57%)
copy server/container/{util/src/test/java/org/apache/james/util/RunnablesTest.java => guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftTest.java} (53%)
copy server/{blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/FileSystemExtension.java => container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Extension.java} (55%)
copy server/container/{util/src/test/java/org/apache/james/util/RunnablesTest.java => guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Test.java} (53%)
copy server/{blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/FileSystemExtension.java => container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftExtension.java} (55%)
copy server/container/{util/src/test/java/org/apache/james/util/RunnablesTest.java => guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftTest.java} (53%)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 02/07: JAMES-2725 Split object storage tests per
implementation
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 9473aac521f91aabd86ce31754f48bac8c1f5670
Author: Antoine Duprat <ad...@linagora.com>
AuthorDate: Tue Apr 16 11:03:29 2019 +0200
JAMES-2725 Split object storage tests per implementation
---
.../james/CassandraRabbitMQJamesServerFixture.java | 43 +++++
.../james/CassandraRabbitMQJamesServerTest.java | 184 ---------------------
.../apache/james/MailsShouldBeWellReceived.java | 67 ++++++++
.../org/apache/james/WithCassandraBlobStore.java | 68 ++++++++
.../apache/james/WithCassandraBlobStoreTest.java | 26 +++
.../apache/james/WithDefaultAwsS3Extension.java | 71 ++++++++
.../org/apache/james/WithDefaultAwsS3Test.java | 41 +++++
.../apache/james/WithDefaultSwiftExtension.java | 71 ++++++++
.../org/apache/james/WithDefaultSwiftTest.java | 41 +++++
.../apache/james/WithEncryptedAwsS3Extension.java | 72 ++++++++
.../org/apache/james/WithEncryptedAwsS3Test.java | 41 +++++
.../apache/james/WithEncryptedSwiftExtension.java | 72 ++++++++
.../org/apache/james/WithEncryptedSwiftTest.java | 42 +++++
.../org/apache/james/JamesServerExtension.java | 5 +
14 files changed, 660 insertions(+), 184 deletions(-)
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java
new file mode 100644
index 0000000..bc5836c
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerFixture.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.apache.james.modules.RabbitMQExtension;
+import org.apache.james.modules.TestJMAPServerModule;
+
+public class CassandraRabbitMQJamesServerFixture {
+
+ private static final int LIMIT_TO_10_MESSAGES = 10;
+
+ private static final JamesServerBuilder.ServerProvider CONFIGURATION_BUILDER =
+ configuration -> GuiceJamesServer
+ .forConfiguration(configuration)
+ .combineWith(CassandraRabbitMQJamesServerMain.MODULES)
+ .overrideWith(new TestJMAPServerModule(LIMIT_TO_10_MESSAGES))
+ .overrideWith(JmapJamesServerContract.DOMAIN_LIST_CONFIGURATION_MODULE);
+
+ public static JamesServerBuilder baseExtensionBuilder() {
+ return new JamesServerBuilder()
+ .extension(new DockerElasticSearchExtension())
+ .extension(new CassandraExtension())
+ .extension(new RabbitMQExtension())
+ .server(CONFIGURATION_BUILDER);
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java
deleted file mode 100644
index 631c93a..0000000
--- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.james;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.awaitility.Duration.ONE_HUNDRED_MILLISECONDS;
-import static org.junit.jupiter.api.TestInstance.Lifecycle;
-
-import org.apache.james.blob.objectstorage.AESPayloadCodec;
-import org.apache.james.blob.objectstorage.DefaultPayloadCodec;
-import org.apache.james.blob.objectstorage.PayloadCodec;
-import org.apache.james.core.Domain;
-import org.apache.james.modules.AwsS3BlobStoreExtension;
-import org.apache.james.modules.RabbitMQExtension;
-import org.apache.james.modules.SwiftBlobStoreExtension;
-import org.apache.james.modules.TestJMAPServerModule;
-import org.apache.james.modules.objectstorage.PayloadCodecFactory;
-import org.apache.james.modules.objectstorage.aws.s3.DockerAwsS3TestRule;
-import org.apache.james.modules.objectstorage.swift.DockerSwiftTestRule;
-import org.apache.james.modules.protocols.ImapGuiceProbe;
-import org.apache.james.modules.protocols.SmtpGuiceProbe;
-import org.apache.james.utils.DataProbeImpl;
-import org.apache.james.utils.IMAPMessageReader;
-import org.apache.james.utils.SMTPMessageSender;
-import org.apache.james.utils.SpoolerProbe;
-import org.awaitility.Awaitility;
-import org.awaitility.core.ConditionFactory;
-import org.junit.jupiter.api.Nested;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-class CassandraRabbitMQJamesServerTest {
-
- interface MailsShouldBeWellReceived {
- String JAMES_SERVER_HOST = "127.0.0.1";
-
- @Test
- default void mailsShouldBeWellReceived(GuiceJamesServer server) throws Exception {
- server.getProbe(DataProbeImpl.class).fluent()
- .addDomain(DOMAIN)
- .addUser(JAMES_USER, PASSWORD);
-
- try (SMTPMessageSender sender = new SMTPMessageSender(Domain.LOCALHOST.asString())) {
- sender.connect(JAMES_SERVER_HOST, server.getProbe(SmtpGuiceProbe.class).getSmtpPort())
- .sendMessage("bob@any.com", JAMES_USER);
- }
-
- CALMLY_AWAIT.until(() -> server.getProbe(SpoolerProbe.class).processingFinished());
-
- try (IMAPMessageReader reader = new IMAPMessageReader()) {
- reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort())
- .login(JAMES_USER, PASSWORD)
- .select(IMAPMessageReader.INBOX)
- .awaitMessage(CALMLY_AWAIT);
- }
- }
- }
-
- interface ContractSuite extends JmapJamesServerContract, MailsShouldBeWellReceived, JamesServerContract {}
-
- private static final String DOMAIN = "domain";
- private static final String JAMES_USER = "james-user@" + DOMAIN;
- private static final String PASSWORD = "secret";
- private static final int LIMIT_TO_10_MESSAGES = 10;
-
- private static final ConditionFactory CALMLY_AWAIT = Awaitility
- .with().pollInterval(ONE_HUNDRED_MILLISECONDS)
- .and().pollDelay(ONE_HUNDRED_MILLISECONDS)
- .await();
-
- private static final JamesServerBuilder.ServerProvider CONFIGURATION_BUILDER =
- configuration -> GuiceJamesServer
- .forConfiguration(configuration)
- .combineWith(CassandraRabbitMQJamesServerMain.MODULES)
- .overrideWith(new TestJMAPServerModule(LIMIT_TO_10_MESSAGES))
- .overrideWith(JmapJamesServerContract.DOMAIN_LIST_CONFIGURATION_MODULE);
-
- @Nested
- @TestInstance(Lifecycle.PER_CLASS)
- class WithEncryptedSwift implements ContractSuite {
- @RegisterExtension
- JamesServerExtension testExtension = baseExtensionBuilder()
- .extension(new SwiftBlobStoreExtension(PayloadCodecFactory.AES256))
- .server(CONFIGURATION_BUILDER)
- .build();
-
- @Test
- void encryptedPayloadShouldBeConfiguredWhenProvidingEncryptedPayloadConfiguration(GuiceJamesServer jamesServer) {
- PayloadCodec payloadCodec = jamesServer.getProbe(DockerSwiftTestRule.TestSwiftBlobStoreProbe.class)
- .getSwiftPayloadCodec();
-
- assertThat(payloadCodec)
- .isInstanceOf(AESPayloadCodec.class);
- }
- }
-
- @Nested
- @TestInstance(Lifecycle.PER_CLASS)
- class WithDefaultSwift implements ContractSuite {
- @RegisterExtension
- JamesServerExtension testExtension = baseExtensionBuilder()
- .extension(new SwiftBlobStoreExtension())
- .build();
-
- @Test
- void defaultPayloadShouldBeByDefault(GuiceJamesServer jamesServer) {
- PayloadCodec payloadCodec = jamesServer.getProbe(DockerSwiftTestRule.TestSwiftBlobStoreProbe.class)
- .getSwiftPayloadCodec();
-
- assertThat(payloadCodec)
- .isInstanceOf(DefaultPayloadCodec.class);
- }
- }
-
- @Nested
- @TestInstance(Lifecycle.PER_CLASS)
- class WithoutSwiftOrAwsS3 implements ContractSuite {
- @RegisterExtension
- JamesServerExtension testExtension = baseExtensionBuilder().build();
- }
-
- @Nested
- @TestInstance(Lifecycle.PER_CLASS)
- class WithEncryptedAwsS3 implements ContractSuite {
- @RegisterExtension
- JamesServerExtension testExtension = baseExtensionBuilder()
- .extension(new AwsS3BlobStoreExtension(PayloadCodecFactory.AES256))
- .server(CONFIGURATION_BUILDER)
- .build();
-
- @Test
- void encryptedPayloadShouldBeConfiguredWhenProvidingEncryptedPayloadConfiguration(GuiceJamesServer jamesServer) {
- PayloadCodec payloadCodec = jamesServer.getProbe(DockerAwsS3TestRule.TestAwsS3BlobStoreProbe.class)
- .getAwsS3PayloadCodec();
-
- assertThat(payloadCodec)
- .isInstanceOf(AESPayloadCodec.class);
- }
- }
-
- @Nested
- @TestInstance(Lifecycle.PER_CLASS)
- class WithDefaultAwsS3 implements ContractSuite {
- @RegisterExtension
- JamesServerExtension testExtension = baseExtensionBuilder()
- .extension(new AwsS3BlobStoreExtension())
- .build();
-
- @Test
- void defaultPayloadShouldBeByDefault(GuiceJamesServer jamesServer) {
- PayloadCodec payloadCodec = jamesServer.getProbe(DockerAwsS3TestRule.TestAwsS3BlobStoreProbe.class)
- .getAwsS3PayloadCodec();
-
- assertThat(payloadCodec)
- .isInstanceOf(DefaultPayloadCodec.class);
- }
- }
-
- private static JamesServerBuilder baseExtensionBuilder() {
- return new JamesServerBuilder()
- .extension(new DockerElasticSearchExtension())
- .extension(new CassandraExtension())
- .extension(new RabbitMQExtension())
- .server(CONFIGURATION_BUILDER);
- }
-}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java
new file mode 100644
index 0000000..16d3f54
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import static org.awaitility.Duration.ONE_HUNDRED_MILLISECONDS;
+
+import org.apache.james.core.Domain;
+import org.apache.james.modules.protocols.ImapGuiceProbe;
+import org.apache.james.modules.protocols.SmtpGuiceProbe;
+import org.apache.james.utils.DataProbeImpl;
+import org.apache.james.utils.IMAPMessageReader;
+import org.apache.james.utils.SMTPMessageSender;
+import org.apache.james.utils.SpoolerProbe;
+import org.awaitility.Awaitility;
+import org.awaitility.core.ConditionFactory;
+import org.junit.jupiter.api.Test;
+
+interface MailsShouldBeWellReceived {
+
+ String JAMES_SERVER_HOST = "127.0.0.1";
+ String DOMAIN = "domain";
+ String JAMES_USER = "james-user@" + DOMAIN;
+ String PASSWORD = "secret";
+ ConditionFactory CALMLY_AWAIT = Awaitility
+ .with().pollInterval(ONE_HUNDRED_MILLISECONDS)
+ .and().pollDelay(ONE_HUNDRED_MILLISECONDS)
+ .await();
+
+
+ @Test
+ default void mailsShouldBeWellReceived(GuiceJamesServer server) throws Exception {
+ server.getProbe(DataProbeImpl.class).fluent()
+ .addDomain(DOMAIN)
+ .addUser(JAMES_USER, PASSWORD);
+
+ try (SMTPMessageSender sender = new SMTPMessageSender(Domain.LOCALHOST.asString())) {
+ sender.connect(JAMES_SERVER_HOST, server.getProbe(SmtpGuiceProbe.class).getSmtpPort())
+ .sendMessage("bob@any.com", JAMES_USER);
+ }
+
+ CALMLY_AWAIT.until(() -> server.getProbe(SpoolerProbe.class).processingFinished());
+
+ try (IMAPMessageReader reader = new IMAPMessageReader()) {
+ reader.connect(JAMES_SERVER_HOST, server.getProbe(ImapGuiceProbe.class).getImapPort())
+ .login(JAMES_USER, PASSWORD)
+ .select(IMAPMessageReader.INBOX)
+ .awaitMessage(CALMLY_AWAIT);
+ }
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStore.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStore.java
new file mode 100644
index 0000000..808c47e
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStore.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+
+public class WithCassandraBlobStore implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback, ParameterResolver {
+
+ private final JamesServerExtension jamesServerExtension;
+
+ WithCassandraBlobStore() {
+ jamesServerExtension = CassandraRabbitMQJamesServerFixture.baseExtensionBuilder().build();
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeAll(context);
+ }
+
+ @Override
+ public void afterAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterAll(context);
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeEach(context);
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterEach(context);
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return (parameterContext.getParameter().getType() == GuiceJamesServer.class);
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return jamesServerExtension.getGuiceJamesServer();
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreTest.java
new file mode 100644
index 0000000..135d2ae
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithCassandraBlobStoreTest.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(WithCassandraBlobStore.class)
+public class WithCassandraBlobStoreTest implements JmapJamesServerContract, MailsShouldBeWellReceived, JamesServerContract {
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Extension.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Extension.java
new file mode 100644
index 0000000..03a035e
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Extension.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.apache.james.modules.AwsS3BlobStoreExtension;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+
+public class WithDefaultAwsS3Extension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback, ParameterResolver {
+
+ private final JamesServerExtension jamesServerExtension;
+
+ WithDefaultAwsS3Extension() {
+ jamesServerExtension = CassandraRabbitMQJamesServerFixture.baseExtensionBuilder()
+ .extension(new AwsS3BlobStoreExtension())
+ .build();
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeAll(context);
+ }
+
+ @Override
+ public void afterAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterAll(context);
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeEach(context);
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterEach(context);
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return (parameterContext.getParameter().getType() == GuiceJamesServer.class);
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return jamesServerExtension.getGuiceJamesServer();
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Test.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Test.java
new file mode 100644
index 0000000..99af818
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultAwsS3Test.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.james.blob.objectstorage.DefaultPayloadCodec;
+import org.apache.james.blob.objectstorage.PayloadCodec;
+import org.apache.james.modules.objectstorage.aws.s3.DockerAwsS3TestRule;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(WithDefaultAwsS3Extension.class)
+public class WithDefaultAwsS3Test implements JmapJamesServerContract, MailsShouldBeWellReceived, JamesServerContract {
+
+ @Test
+ void defaultPayloadShouldBeByDefault(GuiceJamesServer jamesServer) {
+ PayloadCodec payloadCodec = jamesServer.getProbe(DockerAwsS3TestRule.TestAwsS3BlobStoreProbe.class)
+ .getAwsS3PayloadCodec();
+
+ assertThat(payloadCodec)
+ .isInstanceOf(DefaultPayloadCodec.class);
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftExtension.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftExtension.java
new file mode 100644
index 0000000..3e867aa
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftExtension.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.apache.james.modules.SwiftBlobStoreExtension;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+
+public class WithDefaultSwiftExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback, ParameterResolver {
+
+ private final JamesServerExtension jamesServerExtension;
+
+ WithDefaultSwiftExtension() {
+ jamesServerExtension = CassandraRabbitMQJamesServerFixture.baseExtensionBuilder()
+ .extension(new SwiftBlobStoreExtension())
+ .build();
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeAll(context);
+ }
+
+ @Override
+ public void afterAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterAll(context);
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeEach(context);
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterEach(context);
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return (parameterContext.getParameter().getType() == GuiceJamesServer.class);
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return jamesServerExtension.getGuiceJamesServer();
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftTest.java
new file mode 100644
index 0000000..a385484
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithDefaultSwiftTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.james.blob.objectstorage.DefaultPayloadCodec;
+import org.apache.james.blob.objectstorage.PayloadCodec;
+import org.apache.james.modules.objectstorage.swift.DockerSwiftTestRule;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(WithDefaultSwiftExtension.class)
+public class WithDefaultSwiftTest implements JmapJamesServerContract, MailsShouldBeWellReceived, JamesServerContract {
+
+ @Test
+ void defaultPayloadShouldBeByDefault(GuiceJamesServer jamesServer) {
+ PayloadCodec payloadCodec = jamesServer.getProbe(DockerSwiftTestRule.TestSwiftBlobStoreProbe.class)
+ .getSwiftPayloadCodec();
+
+ assertThat(payloadCodec)
+ .isInstanceOf(DefaultPayloadCodec.class);
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Extension.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Extension.java
new file mode 100644
index 0000000..c4927a2
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Extension.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.apache.james.modules.AwsS3BlobStoreExtension;
+import org.apache.james.modules.objectstorage.PayloadCodecFactory;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+
+public class WithEncryptedAwsS3Extension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback, ParameterResolver {
+
+ private final JamesServerExtension jamesServerExtension;
+
+ WithEncryptedAwsS3Extension() {
+ jamesServerExtension = CassandraRabbitMQJamesServerFixture.baseExtensionBuilder()
+ .extension(new AwsS3BlobStoreExtension(PayloadCodecFactory.AES256))
+ .build();
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeAll(context);
+ }
+
+ @Override
+ public void afterAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterAll(context);
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeEach(context);
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterEach(context);
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return (parameterContext.getParameter().getType() == GuiceJamesServer.class);
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return jamesServerExtension.getGuiceJamesServer();
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Test.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Test.java
new file mode 100644
index 0000000..312a8e0
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedAwsS3Test.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.james.blob.objectstorage.AESPayloadCodec;
+import org.apache.james.blob.objectstorage.PayloadCodec;
+import org.apache.james.modules.objectstorage.aws.s3.DockerAwsS3TestRule;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(WithEncryptedAwsS3Extension.class)
+public class WithEncryptedAwsS3Test implements JmapJamesServerContract, MailsShouldBeWellReceived, JamesServerContract {
+
+ @Test
+ void encryptedPayloadShouldBeConfiguredWhenProvidingEncryptedPayloadConfiguration(GuiceJamesServer jamesServer) {
+ PayloadCodec payloadCodec = jamesServer.getProbe(DockerAwsS3TestRule.TestAwsS3BlobStoreProbe.class)
+ .getAwsS3PayloadCodec();
+
+ assertThat(payloadCodec)
+ .isInstanceOf(AESPayloadCodec.class);
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftExtension.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftExtension.java
new file mode 100644
index 0000000..422a281
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftExtension.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import org.apache.james.modules.SwiftBlobStoreExtension;
+import org.apache.james.modules.objectstorage.PayloadCodecFactory;
+import org.junit.jupiter.api.extension.AfterAllCallback;
+import org.junit.jupiter.api.extension.AfterEachCallback;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.BeforeEachCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.ParameterResolver;
+
+public class WithEncryptedSwiftExtension implements BeforeAllCallback, AfterAllCallback, BeforeEachCallback, AfterEachCallback, ParameterResolver {
+
+ private final JamesServerExtension jamesServerExtension;
+
+ WithEncryptedSwiftExtension() {
+ jamesServerExtension = CassandraRabbitMQJamesServerFixture.baseExtensionBuilder()
+ .extension(new SwiftBlobStoreExtension(PayloadCodecFactory.AES256))
+ .build();
+ }
+
+ @Override
+ public void beforeAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeAll(context);
+ }
+
+ @Override
+ public void afterAll(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterAll(context);
+ }
+
+ @Override
+ public void beforeEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.beforeEach(context);
+ }
+
+ @Override
+ public void afterEach(ExtensionContext context) throws Exception {
+ jamesServerExtension.afterEach(context);
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return (parameterContext.getParameter().getType() == GuiceJamesServer.class);
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return jamesServerExtension.getGuiceJamesServer();
+ }
+}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftTest.java
new file mode 100644
index 0000000..74b1fd2
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/WithEncryptedSwiftTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.apache.james.blob.objectstorage.AESPayloadCodec;
+import org.apache.james.blob.objectstorage.PayloadCodec;
+import org.apache.james.modules.objectstorage.swift.DockerSwiftTestRule;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+@ExtendWith(WithEncryptedSwiftExtension.class)
+public class WithEncryptedSwiftTest implements JmapJamesServerContract, MailsShouldBeWellReceived, JamesServerContract {
+
+ @Test
+ void encryptedPayloadShouldBeConfiguredWhenProvidingEncryptedPayloadConfiguration(GuiceJamesServer jamesServer) {
+ PayloadCodec payloadCodec = jamesServer.getProbe(DockerSwiftTestRule.TestSwiftBlobStoreProbe.class)
+ .getSwiftPayloadCodec();
+
+ assertThat(payloadCodec)
+ .isInstanceOf(AESPayloadCodec.class);
+ }
+
+}
diff --git a/server/container/guice/guice-common/src/test/java/org/apache/james/JamesServerExtension.java b/server/container/guice/guice-common/src/test/java/org/apache/james/JamesServerExtension.java
index 99c42b2..3469871 100644
--- a/server/container/guice/guice-common/src/test/java/org/apache/james/JamesServerExtension.java
+++ b/server/container/guice/guice-common/src/test/java/org/apache/james/JamesServerExtension.java
@@ -47,6 +47,7 @@ public class JamesServerExtension implements BeforeAllCallback, BeforeEachCallba
private final RegistrableExtension registrableExtension;
private final boolean autoStart;
private final AwaitCondition awaitCondition;
+
private GuiceJamesServer guiceJamesServer;
JamesServerExtension(RegistrableExtension registrableExtension, ThrowingFunction<File, GuiceJamesServer> serverSupplier,
@@ -58,6 +59,10 @@ public class JamesServerExtension implements BeforeAllCallback, BeforeEachCallba
this.awaitCondition = awaitCondition;
}
+ public GuiceJamesServer getGuiceJamesServer() {
+ return guiceJamesServer;
+ }
+
@Override
public void beforeAll(ExtensionContext extensionContext) throws Exception {
registrableExtension.beforeAll(extensionContext);
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 01/07: JAMES-2725 Remove length from Blob API This
means that the S3 implementation should implement its own way to put blobs
without knowing the length
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 9e2c0cfef679a6e92e0bd92abaf0607de7948ef3
Author: Antoine Duprat <ad...@linagora.com>
AuthorDate: Fri Apr 5 11:37:07 2019 +0200
JAMES-2725 Remove length from Blob API
This means that the S3 implementation should implement its own way to put blobs without knowing the length
---
.../java/org/apache/james/blob/api/BlobStore.java | 2 +-
.../apache/james/blob/api/MetricableBlobStore.java | 4 +-
.../main/java/org/apache/james/blob/api/Store.java | 28 +------
.../apache/james/blob/api/BlobStoreContract.java | 6 +-
.../james/blob/api/FixedLengthInputStreamTest.java | 63 ----------------
.../blob/api/MetricableBlobStoreContract.java | 4 +-
.../james/blob/cassandra/CassandraBlobsDAO.java | 2 +-
.../blob/cassandra/CassandraBlobsDAOTest.java | 2 +-
.../apache/james/blob/memory/MemoryBlobStore.java | 2 +-
server/blob/blob-objectstorage/pom.xml | 5 ++
.../blob/objectstorage/ObjectStorageBlobsDAO.java | 17 +++--
.../ObjectStorageBlobsDAOBuilder.java | 19 ++++-
.../james/blob/objectstorage/PutBlobFunction.java | 37 ++++++++++
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 85 +++++++++++++++++++++-
.../objectstorage/ObjectStorageBlobsDAOTest.java | 2 +-
.../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 4 +-
.../apache/james/blob/union/UnionBlobStore.java | 8 +-
.../james/blob/union/UnionBlobStoreTest.java | 14 ++--
.../apache/james/blob/mail/MimeMessageStore.java | 6 +-
.../ObjectStorageDependenciesModule.java | 14 ++++
.../james/CassandraRabbitMQJamesServerTest.java | 10 +--
.../james/webadmin/vault/routes/ExportService.java | 2 +-
22 files changed, 205 insertions(+), 131 deletions(-)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java
index d22be29..762d916 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java
@@ -27,7 +27,7 @@ public interface BlobStore {
Mono<BlobId> save(byte[] data);
- Mono<BlobId> save(InputStream data, long contentLength);
+ Mono<BlobId> save(InputStream data);
Mono<byte[]> readBytes(BlobId blobId);
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java
index b51e37b..4ed7b17 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java
@@ -54,9 +54,9 @@ public class MetricableBlobStore implements BlobStore {
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
return metricFactory
- .runPublishingTimerMetric(SAVE_INPUT_STREAM_TIMER_NAME, blobStoreImpl.save(data, contentLength));
+ .runPublishingTimerMetric(SAVE_INPUT_STREAM_TIMER_NAME, blobStoreImpl.save(data));
}
@Override
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/Store.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/Store.java
index 3cd4afa..07a5611 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/Store.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/Store.java
@@ -26,7 +26,6 @@ import java.util.stream.Stream;
import org.apache.commons.lang3.tuple.Pair;
-import com.google.common.base.Preconditions;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.function.Tuple2;
@@ -67,7 +66,7 @@ public interface Store<T, I> {
class Impl<T, I extends BlobPartsId> implements Store<T, I> {
public interface Encoder<T> {
- Stream<Pair<BlobType, FixedLengthInputStream>> encode(T t);
+ Stream<Pair<BlobType, InputStream>> encode(T t);
}
public interface Decoder<T> {
@@ -94,9 +93,9 @@ public interface Store<T, I> {
.map(idFactory::generate);
}
- private Mono<Tuple2<BlobType, BlobId>> saveEntry(Pair<BlobType, FixedLengthInputStream> entry) {
+ private Mono<Tuple2<BlobType, BlobId>> saveEntry(Pair<BlobType, InputStream> entry) {
return Mono.just(entry.getLeft())
- .zipWith(blobStore.save(entry.getRight().getInputStream(), entry.getRight().getContentLength()));
+ .zipWith(blobStore.save(entry.getRight()));
}
@Override
@@ -111,25 +110,4 @@ public interface Store<T, I> {
.map(decoder::decode);
}
}
-
- class FixedLengthInputStream {
-
- private final InputStream inputStream;
- private final long contentLength;
-
- public FixedLengthInputStream(InputStream inputStream, long contentLength) {
- Preconditions.checkNotNull(inputStream, "'inputStream' is mandatory");
- Preconditions.checkArgument(contentLength >= 0, "'contentLength' should be greater than or equal to 0");
- this.inputStream = inputStream;
- this.contentLength = contentLength;
- }
-
- public InputStream getInputStream() {
- return inputStream;
- }
-
- public long getContentLength() {
- return contentLength;
- }
- }
}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
index dbb9863..5cb31be 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
@@ -56,7 +56,7 @@ public interface BlobStoreContract {
@Test
default void saveShouldThrowWhenNullInputStream() {
- assertThatThrownBy(() -> testee().save((InputStream) null, 0).block())
+ assertThatThrownBy(() -> testee().save((InputStream) null).block())
.isInstanceOf(NullPointerException.class);
}
@@ -80,7 +80,7 @@ public interface BlobStoreContract {
@Test
default void saveShouldSaveEmptyInputStream() {
- BlobId blobId = testee().save(new ByteArrayInputStream(EMPTY_BYTEARRAY), EMPTY_BYTEARRAY.length).block();
+ BlobId blobId = testee().save(new ByteArrayInputStream(EMPTY_BYTEARRAY)).block();
byte[] bytes = testee().readBytes(blobId).block();
@@ -104,7 +104,7 @@ public interface BlobStoreContract {
@Test
default void saveShouldReturnBlobIdOfInputStream() {
BlobId blobId =
- testee().save(new ByteArrayInputStream(SHORT_BYTEARRAY), SHORT_BYTEARRAY.length).block();
+ testee().save(new ByteArrayInputStream(SHORT_BYTEARRAY)).block();
assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/FixedLengthInputStreamTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/FixedLengthInputStreamTest.java
deleted file mode 100644
index c442b0c..0000000
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/FixedLengthInputStreamTest.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.james.blob.api;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-import java.io.ByteArrayInputStream;
-import java.nio.charset.StandardCharsets;
-
-import org.junit.jupiter.api.Test;
-
-class FixedLengthInputStreamTest {
-
- @Test
- void fixedLengthInputStreamShouldThrowWhenInputStreamIsNull() {
- assertThatThrownBy(() -> new Store.FixedLengthInputStream(null, 0))
- .isInstanceOf(NullPointerException.class)
- .hasMessage("'inputStream' is mandatory");
- }
-
- @Test
- void fixedLengthInputStreamShouldThrowWhenContentLengthIsNegative() {
- assertThatThrownBy(() -> new Store.FixedLengthInputStream(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)), -1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("'contentLength' should be greater than or equal to 0");
- }
-
- @Test
- void lengthShouldBeStored() {
- int contentLength = 1;
-
- Store.FixedLengthInputStream testee = new Store.FixedLengthInputStream(new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8)), contentLength);
-
- assertThat(testee.getContentLength()).isEqualTo(contentLength);
- }
-
- @Test
- void inputStreamShouldBeStored() {
- ByteArrayInputStream inputStream = new ByteArrayInputStream("".getBytes(StandardCharsets.UTF_8));
-
- Store.FixedLengthInputStream testee = new Store.FixedLengthInputStream(inputStream, 1);
-
- assertThat(testee.getInputStream()).hasSameContentAs(inputStream);
- }
-}
\ No newline at end of file
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/MetricableBlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/MetricableBlobStoreContract.java
index 7ed9b7e..ce57b53 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/MetricableBlobStoreContract.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/MetricableBlobStoreContract.java
@@ -75,8 +75,8 @@ public interface MetricableBlobStoreContract extends BlobStoreContract {
@Test
default void saveInputStreamShouldPublishSaveInputStreamTimerMetrics() {
- testee().save(new ByteArrayInputStream(BYTES_CONTENT), BYTES_CONTENT.length).block();
- testee().save(new ByteArrayInputStream(BYTES_CONTENT), BYTES_CONTENT.length).block();
+ testee().save(new ByteArrayInputStream(BYTES_CONTENT)).block();
+ testee().save(new ByteArrayInputStream(BYTES_CONTENT)).block();
assertThat(metricsTestExtension.getMetricFactory().executionTimesFor(SAVE_INPUT_STREAM_TIMER_NAME))
.hasSize(2);
diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobsDAO.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobsDAO.java
index b637601..444c55a 100644
--- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobsDAO.java
+++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobsDAO.java
@@ -212,7 +212,7 @@ public class CassandraBlobsDAO implements BlobStore {
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
Preconditions.checkNotNull(data);
return Mono.fromCallable(() -> IOUtils.toByteArray(data))
.flatMap(this::saveAsMono);
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobsDAOTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobsDAOTest.java
index 9831bed..a372a35 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobsDAOTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobsDAOTest.java
@@ -81,7 +81,7 @@ public class CassandraBlobsDAOTest implements MetricableBlobStoreContract {
@Test
void blobStoreShouldSupport100MBBlob() {
- BlobId blobId = testee.save(new ZeroedInputStream(100_000_000), 100_000_000).block();
+ BlobId blobId = testee.save(new ZeroedInputStream(100_000_000)).block();
InputStream bytes = testee.read(blobId);
assertThat(bytes).hasSameContentAs(new ZeroedInputStream(100_000_000));
}
diff --git a/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStore.java b/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStore.java
index c47d83f..21f47ef 100644
--- a/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStore.java
+++ b/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStore.java
@@ -57,7 +57,7 @@ public class MemoryBlobStore implements BlobStore {
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
Preconditions.checkNotNull(data);
try {
byte[] bytes = IOUtils.toByteArray(data);
diff --git a/server/blob/blob-objectstorage/pom.xml b/server/blob/blob-objectstorage/pom.xml
index fe8481d..c1bbf81 100644
--- a/server/blob/blob-objectstorage/pom.xml
+++ b/server/blob/blob-objectstorage/pom.xml
@@ -67,6 +67,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.11.532</version>
+ </dependency>
+ <dependency>
<groupId>com.google.crypto.tink</groupId>
<artifactId>tink</artifactId>
<version>1.2.0</version>
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAO.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAO.java
index 6000680..113d6b4 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAO.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAO.java
@@ -54,13 +54,17 @@ public class ObjectStorageBlobsDAO implements BlobStore {
private final ContainerName containerName;
private final org.jclouds.blobstore.BlobStore blobStore;
+ private final PutBlobFunction putBlobFunction;
private final PayloadCodec payloadCodec;
ObjectStorageBlobsDAO(ContainerName containerName, BlobId.Factory blobIdFactory,
- org.jclouds.blobstore.BlobStore blobStore, PayloadCodec payloadCodec) {
+ org.jclouds.blobstore.BlobStore blobStore,
+ PutBlobFunction putBlobFunction,
+ PayloadCodec payloadCodec) {
this.blobIdFactory = blobIdFactory;
this.containerName = containerName;
this.blobStore = blobStore;
+ this.putBlobFunction = putBlobFunction;
this.payloadCodec = payloadCodec;
}
@@ -89,15 +93,15 @@ public class ObjectStorageBlobsDAO implements BlobStore {
@Override
public Mono<BlobId> save(byte[] data) {
- return save(new ByteArrayInputStream(data), data.length);
+ return save(new ByteArrayInputStream(data));
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
Preconditions.checkNotNull(data);
BlobId tmpId = blobIdFactory.randomId();
- return save(data, contentLength, tmpId)
+ return save(data, tmpId)
.flatMap(id -> updateBlobId(tmpId, id));
}
@@ -109,16 +113,15 @@ public class ObjectStorageBlobsDAO implements BlobStore {
.thenReturn(to);
}
- private Mono<BlobId> save(InputStream data, long contentLength, BlobId id) {
+ private Mono<BlobId> save(InputStream data, BlobId id) {
String containerName = this.containerName.value();
HashingInputStream hashingInputStream = new HashingInputStream(Hashing.sha256(), data);
Payload payload = payloadCodec.write(hashingInputStream);
Blob blob = blobStore.blobBuilder(id.asString())
.payload(payload.getPayload())
- .contentLength(payload.getLength().orElse(contentLength))
.build();
- return Mono.fromCallable(() -> blobStore.putBlob(containerName, blob))
+ return Mono.fromRunnable(() -> putBlobFunction.putBlob(blob))
.then(Mono.fromCallable(() -> blobIdFactory.from(hashingInputStream.hash().toString())));
}
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOBuilder.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOBuilder.java
index 4e5f941..f6aa7ff 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOBuilder.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOBuilder.java
@@ -50,12 +50,14 @@ public class ObjectStorageBlobsDAOBuilder {
private final ContainerName containerName;
private final BlobId.Factory blobIdFactory;
private Optional<PayloadCodec> payloadCodec;
+ private Optional<PutBlobFunction> putBlob;
public ReadyToBuild(Supplier<BlobStore> supplier, BlobId.Factory blobIdFactory, ContainerName containerName) {
this.blobIdFactory = blobIdFactory;
this.containerName = containerName;
this.payloadCodec = Optional.empty();
this.supplier = supplier;
+ this.putBlob = Optional.empty();
}
public ReadyToBuild payloadCodec(PayloadCodec payloadCodec) {
@@ -68,11 +70,26 @@ public class ObjectStorageBlobsDAOBuilder {
return this;
}
+ public ReadyToBuild putBlob(Optional<PutBlobFunction> putBlob) {
+ this.putBlob = putBlob;
+ return this;
+ }
+
public ObjectStorageBlobsDAO build() {
Preconditions.checkState(containerName != null);
Preconditions.checkState(blobIdFactory != null);
- return new ObjectStorageBlobsDAO(containerName, blobIdFactory, supplier.get(), payloadCodec.orElse(PayloadCodec.DEFAULT_CODEC));
+ BlobStore blobStore = supplier.get();
+
+ return new ObjectStorageBlobsDAO(containerName,
+ blobIdFactory,
+ blobStore,
+ putBlob.orElse(defaultPutBlob(blobStore)),
+ payloadCodec.orElse(PayloadCodec.DEFAULT_CODEC));
+ }
+
+ private PutBlobFunction defaultPutBlob(BlobStore blobStore) {
+ return (blob) -> blobStore.putBlob(containerName.value(), blob);
}
@VisibleForTesting
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/PutBlobFunction.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/PutBlobFunction.java
new file mode 100644
index 0000000..ac58aef
--- /dev/null
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/PutBlobFunction.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.james.blob.objectstorage;
+
+import org.jclouds.blobstore.domain.Blob;
+
+/**
+ * Implementations may have specific behaviour when uploading a blob,
+ * such cases are not well handled by jClouds.
+ *
+ * For example:
+ * AWS S3 need a length while uploading with jClouds
+ * whereas you don't need one by using the S3 client.
+ *
+ */
+@FunctionalInterface
+public interface PutBlobFunction {
+
+ void putBlob(Blob blob);
+}
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index be5eac5..7940ef0 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -19,27 +19,108 @@
package org.apache.james.blob.objectstorage.aws;
+import java.io.File;
+import java.io.IOException;
+import java.util.Optional;
import java.util.Properties;
+import java.util.UUID;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.function.Supplier;
+import org.apache.commons.io.FileUtils;
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
+import org.apache.james.blob.objectstorage.PutBlobFunction;
+import org.apache.james.util.Size;
+import org.apache.james.util.concurrent.NamedThreadFactory;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.transfer.TransferManager;
+import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
public class AwsS3ObjectStorage {
- private static final Iterable<Module> JCLOUDS_MODULES =
- ImmutableSet.of(new SLF4JLoggingModule());
+ private static final Iterable<Module> JCLOUDS_MODULES = ImmutableSet.of(new SLF4JLoggingModule());
+ public static final int MAX_THREADS = 5;
+ private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
+ private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false;
+ private static Size MULTIPART_UPLOAD_THRESHOLD;
+
+ static {
+ try {
+ MULTIPART_UPLOAD_THRESHOLD = Size.parse("5M");
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
public static ObjectStorageBlobsDAOBuilder.RequireContainerName daoBuilder(AwsS3AuthConfiguration configuration) {
return ObjectStorageBlobsDAOBuilder.forBlobStore(new BlobStoreBuilder(configuration));
}
+ public static Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration) {
+ return Optional.of((blob) -> {
+ File file = null;
+ try {
+ file = File.createTempFile(UUID.randomUUID().toString(), ".tmp");
+ FileUtils.copyToFile(blob.getPayload().openStream(), file);
+
+ put(blobIdFactory, containerName, configuration, blob, file);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ if (file != null) {
+ FileUtils.deleteQuietly(file);
+ }
+ }
+ });
+ }
+
+ private static void put(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
+ try {
+ PutObjectRequest request = new PutObjectRequest(containerName.value(),
+ blob.getMetadata().getName(),
+ file);
+
+ getTransferManager(configuration)
+ .upload(request)
+ .waitForUploadResult();
+ } catch (AmazonClientException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
+ AmazonS3 amazonS3 = AmazonS3ClientBuilder
+ .standard()
+ .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(configuration.getAccessKeyId(), configuration.getSecretKey())))
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(configuration.getEndpoint(), null))
+ .build();
+
+ return TransferManagerBuilder
+ .standard()
+ .withS3Client(amazonS3)
+ .withMultipartUploadThreshold(MULTIPART_UPLOAD_THRESHOLD.getValue())
+ .withExecutorFactory(() -> EXECUTOR_SERVICE)
+ .withShutDownThreadPools(DO_NOT_SHUTDOWN_THREAD_POOL)
+ .build();
+ }
+
private static class BlobStoreBuilder implements Supplier<BlobStore> {
private final AwsS3AuthConfiguration configuration;
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java
index b1ec0c4..caabb42 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java
@@ -189,7 +189,7 @@ public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract {
@Test
void saveInputStreamShouldNotCompleteWhenDoesNotAwait() {
Mono<BlobId> blobIdFuture = testee
- .save(new ByteArrayInputStream(BIG_STRING.getBytes(StandardCharsets.UTF_8)), BIG_STRING.length())
+ .save(new ByteArrayInputStream(BIG_STRING.getBytes(StandardCharsets.UTF_8)))
.subscribeOn(Schedulers.elastic());
assertThat(blobIdFuture.toFuture()).isNotCompleted();
}
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
index 147c130..9ecda9f 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
@@ -24,6 +24,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.UUID;
import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.blob.api.TestBlobId;
import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAO;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
@@ -78,7 +79,8 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
ObjectStorageBlobsDAOBuilder.ReadyToBuild builder = ObjectStorageBlobsDAO
.builder(configuration)
.container(containerName)
- .blobIdFactory(new HashBlobId.Factory());
+ .blobIdFactory(new HashBlobId.Factory())
+ .putBlob(AwsS3ObjectStorage.putBlob(new TestBlobId.Factory(), containerName, configuration));
assertBlobsDAOCanStoreAndRetrieve(builder);
}
diff --git a/server/blob/blob-union/src/main/java/org/apache/james/blob/union/UnionBlobStore.java b/server/blob/blob-union/src/main/java/org/apache/james/blob/union/UnionBlobStore.java
index 07d0830..1991196 100644
--- a/server/blob/blob-union/src/main/java/org/apache/james/blob/union/UnionBlobStore.java
+++ b/server/blob/blob-union/src/main/java/org/apache/james/blob/union/UnionBlobStore.java
@@ -105,14 +105,14 @@ public class UnionBlobStore implements BlobStore {
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
try {
return saveToCurrentFallbackIfFails(
- Mono.defer(() -> currentBlobStore.save(data, contentLength)),
- () -> Mono.defer(() -> legacyBlobStore.save(data, contentLength)));
+ Mono.defer(() -> currentBlobStore.save(data)),
+ () -> Mono.defer(() -> legacyBlobStore.save(data)));
} catch (Exception e) {
LOGGER.error("exception directly happens while saving InputStream data, fall back to legacy blob store", e);
- return legacyBlobStore.save(data, contentLength);
+ return legacyBlobStore.save(data);
}
}
diff --git a/server/blob/blob-union/src/test/java/org/apache/james/blob/union/UnionBlobStoreTest.java b/server/blob/blob-union/src/test/java/org/apache/james/blob/union/UnionBlobStoreTest.java
index d567a94..1956350 100644
--- a/server/blob/blob-union/src/test/java/org/apache/james/blob/union/UnionBlobStoreTest.java
+++ b/server/blob/blob-union/src/test/java/org/apache/james/blob/union/UnionBlobStoreTest.java
@@ -66,7 +66,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
return Mono.error(new RuntimeException("broken everywhere"));
}
@@ -100,7 +100,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
}
@Override
- public Mono<BlobId> save(InputStream data, long contentLength) {
+ public Mono<BlobId> save(InputStream data) {
throw new RuntimeException("broken everywhere");
}
@@ -176,7 +176,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
.current(new ThrowingBlobStore())
.legacy(legacyBlobStore)
.build();
- BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT), BLOB_CONTENT.length).block();
+ BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT)).block();
SoftAssertions.assertSoftly(softly -> {
softly.assertThat(unionBlobStore.read(blobId))
@@ -214,7 +214,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
.current(new FailingBlobStore())
.legacy(legacyBlobStore)
.build();
- BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT), BLOB_CONTENT.length).block();
+ BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT)).block();
SoftAssertions.assertSoftly(softly -> {
softly.assertThat(unionBlobStore.read(blobId))
@@ -297,7 +297,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
return Stream.of(
blobStore -> blobStore.save(BLOB_CONTENT),
blobStore -> blobStore.save(STRING_CONTENT),
- blobStore -> blobStore.save(new ByteArrayInputStream(BLOB_CONTENT), BLOB_CONTENT.length),
+ blobStore -> blobStore.save(new ByteArrayInputStream(BLOB_CONTENT)),
blobStore -> blobStore.readBytes(BLOB_ID_FACTORY.randomId()));
}
@@ -423,7 +423,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
@Test
void saveInputStreamShouldWriteToCurrent() {
- BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT), BLOB_CONTENT.length).block();
+ BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT)).block();
assertThat(currentBlobStore.readBytes(blobId).block())
.isEqualTo(BLOB_CONTENT);
@@ -431,7 +431,7 @@ class UnionBlobStoreTest implements BlobStoreContract {
@Test
void saveInputStreamShouldNotWriteToLegacy() {
- BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT), BLOB_CONTENT.length).block();
+ BlobId blobId = unionBlobStore.save(new ByteArrayInputStream(BLOB_CONTENT)).block();
assertThatThrownBy(() -> legacyBlobStore.readBytes(blobId).block())
.isInstanceOf(ObjectStoreException.class);
diff --git a/server/blob/mail-store/src/main/java/org/apache/james/blob/mail/MimeMessageStore.java b/server/blob/mail-store/src/main/java/org/apache/james/blob/mail/MimeMessageStore.java
index 506b895..12683f9 100644
--- a/server/blob/mail-store/src/main/java/org/apache/james/blob/mail/MimeMessageStore.java
+++ b/server/blob/mail-store/src/main/java/org/apache/james/blob/mail/MimeMessageStore.java
@@ -69,15 +69,15 @@ public class MimeMessageStore {
static class MimeMessageEncoder implements Store.Impl.Encoder<MimeMessage> {
@Override
- public Stream<Pair<BlobType, Store.FixedLengthInputStream>> encode(MimeMessage message) {
+ public Stream<Pair<BlobType, InputStream>> encode(MimeMessage message) {
try {
byte[] messageAsArray = messageToArray(message);
int bodyStartOctet = computeBodyStartOctet(messageAsArray);
byte[] headerBytes = getHeaderBytes(messageAsArray, bodyStartOctet);
byte[] bodyBytes = getBodyBytes(messageAsArray, bodyStartOctet);
return Stream.of(
- Pair.of(HEADER_BLOB_TYPE, new Store.FixedLengthInputStream(new ByteArrayInputStream(headerBytes), headerBytes.length)),
- Pair.of(BODY_BLOB_TYPE, new Store.FixedLengthInputStream(new ByteArrayInputStream(bodyBytes), bodyBytes.length)));
+ Pair.of(HEADER_BLOB_TYPE, new ByteArrayInputStream(headerBytes)),
+ Pair.of(BODY_BLOB_TYPE, new ByteArrayInputStream(bodyBytes)));
} catch (MessagingException | IOException e) {
throw new RuntimeException(e);
}
diff --git a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
index 0d725ed..23c40b9 100644
--- a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
+++ b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
@@ -21,6 +21,7 @@ package org.apache.james.modules.objectstorage;
import java.io.FileNotFoundException;
import java.time.Duration;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
@@ -32,6 +33,7 @@ import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAO;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
+import org.apache.james.blob.objectstorage.PutBlobFunction;
import org.apache.james.blob.objectstorage.aws.AwsS3AuthConfiguration;
import org.apache.james.blob.objectstorage.aws.AwsS3ObjectStorage;
import org.apache.james.modules.mailbox.ConfigurationComponent;
@@ -67,6 +69,7 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
.container(configuration.getNamespace())
.blobIdFactory(blobIdFactory)
.payloadCodec(configuration.getPayloadCodec())
+ .putBlob(putBlob(blobIdFactory, configuration))
.build();
dao.createContainer(configuration.getNamespace()).block(Duration.ofMinutes(1));
return dao;
@@ -82,4 +85,15 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
}
+ private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration) {
+ switch (configuration.getProvider()) {
+ case SWIFT:
+ return Optional.empty();
+ case AWSS3:
+ return AwsS3ObjectStorage.putBlob(blobIdFactory, configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
+ }
+ throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
+
+ }
+
}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java
index 4c3c8a0..631c93a 100644
--- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/CassandraRabbitMQJamesServerTest.java
@@ -94,7 +94,7 @@ class CassandraRabbitMQJamesServerTest {
.overrideWith(JmapJamesServerContract.DOMAIN_LIST_CONFIGURATION_MODULE);
@Nested
- @TestInstance(Lifecycle.PER_METHOD)
+ @TestInstance(Lifecycle.PER_CLASS)
class WithEncryptedSwift implements ContractSuite {
@RegisterExtension
JamesServerExtension testExtension = baseExtensionBuilder()
@@ -113,7 +113,7 @@ class CassandraRabbitMQJamesServerTest {
}
@Nested
- @TestInstance(Lifecycle.PER_METHOD)
+ @TestInstance(Lifecycle.PER_CLASS)
class WithDefaultSwift implements ContractSuite {
@RegisterExtension
JamesServerExtension testExtension = baseExtensionBuilder()
@@ -131,14 +131,14 @@ class CassandraRabbitMQJamesServerTest {
}
@Nested
- @TestInstance(Lifecycle.PER_METHOD)
+ @TestInstance(Lifecycle.PER_CLASS)
class WithoutSwiftOrAwsS3 implements ContractSuite {
@RegisterExtension
JamesServerExtension testExtension = baseExtensionBuilder().build();
}
@Nested
- @TestInstance(Lifecycle.PER_METHOD)
+ @TestInstance(Lifecycle.PER_CLASS)
class WithEncryptedAwsS3 implements ContractSuite {
@RegisterExtension
JamesServerExtension testExtension = baseExtensionBuilder()
@@ -157,7 +157,7 @@ class CassandraRabbitMQJamesServerTest {
}
@Nested
- @TestInstance(Lifecycle.PER_METHOD)
+ @TestInstance(Lifecycle.PER_CLASS)
class WithDefaultAwsS3 implements ContractSuite {
@RegisterExtension
JamesServerExtension testExtension = baseExtensionBuilder()
diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java
index a3ca2ce..14c100c 100644
--- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java
+++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/main/java/org/apache/james/webadmin/vault/routes/ExportService.java
@@ -74,7 +74,7 @@ class ExportService {
try (FileBackedOutputStream fileOutputStream = new FileBackedOutputStream(FileUtils.ONE_MB_BI.intValue())) {
zipper.zip(contentLoader(user), messages.toStream(), fileOutputStream);
ByteSource byteSource = fileOutputStream.asByteSource();
- return blobStore.save(byteSource.openStream(), byteSource.size()).block();
+ return blobStore.save(byteSource.openStream()).block();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 04/07: JAMES-2725 add retry on exception for s3
upload
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 8e73a1d2429fd4ffb2bda0dfa697bed49213189b
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Tue May 7 13:40:47 2019 +0200
JAMES-2725 add retry on exception for s3 upload
---
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 18 +++++++++++++++---
.../org/apache/james/MailsShouldBeWellReceived.java | 1 -
2 files changed, 15 insertions(+), 4 deletions(-)
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index 56d58a3..f00be8b 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -61,8 +61,9 @@ public class AwsS3ObjectStorage {
public static final int MAX_THREADS = 5;
private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false;
- public static final int MAX_UPLOAD_THREADS = 5;
private static final int MAX_ERROR_RETRY = 5;
+ private static final int FIRST_TRY = 0;
+ private static final int MAX_RETRY_ON_EXCEPTION = 3;
public static Size MULTIPART_UPLOAD_THRESHOLD;
static {
@@ -83,8 +84,7 @@ public class AwsS3ObjectStorage {
try {
file = File.createTempFile(UUID.randomUUID().toString(), ".tmp");
FileUtils.copyToFile(blob.getPayload().openStream(), file);
-
- put(blobIdFactory, containerName, configuration, blob, file);
+ putWithRetry(blobIdFactory, containerName, configuration, blob, file, FIRST_TRY);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
@@ -95,6 +95,18 @@ public class AwsS3ObjectStorage {
});
}
+ private static void putWithRetry(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
+ try {
+ put(blobIdFactory, containerName, configuration, blob, file);
+ } catch (RuntimeException e) {
+ if (tried < MAX_RETRY_ON_EXCEPTION) {
+ putWithRetry(blobIdFactory, containerName, configuration, blob, file, tried + 1);
+ } else {
+ throw e;
+ }
+ }
+ }
+
private static void put(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
try {
PutObjectRequest request = new PutObjectRequest(containerName.value(),
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java
index 16d3f54..9c8adbf 100644
--- a/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/test/java/org/apache/james/MailsShouldBeWellReceived.java
@@ -43,7 +43,6 @@ interface MailsShouldBeWellReceived {
.and().pollDelay(ONE_HUNDRED_MILLISECONDS)
.await();
-
@Test
default void mailsShouldBeWellReceived(GuiceJamesServer server) throws Exception {
server.getProbe(DataProbeImpl.class).fluent()
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 06/07: JAMES-2725 Use injection for AwsS3Object
storage for managing its thread pool lifecycle.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 712f005be906c4eefdf3158960407b4a58dc5357
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Thu May 9 18:02:48 2019 +0200
JAMES-2725 Use injection for AwsS3Object storage for managing its thread pool lifecycle.
---
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 30 +++++++++++++++++-----
.../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 10 +++++++-
.../ObjectStorageDependenciesModule.java | 11 +++++---
3 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index fbf48a4..5454af0 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -28,6 +28,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Supplier;
+import javax.annotation.PreDestroy;
+import javax.inject.Inject;
+
import org.apache.commons.io.FileUtils;
import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
@@ -51,14 +54,14 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
public class AwsS3ObjectStorage {
private static final Iterable<Module> JCLOUDS_MODULES = ImmutableSet.of(new SLF4JLoggingModule());
- public static final int MAX_THREADS = 5;
- private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
+ public static final int MAX_THREADS = 5;
private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false;
private static final int MAX_ERROR_RETRY = 5;
private static final int FIRST_TRY = 0;
@@ -73,11 +76,24 @@ public class AwsS3ObjectStorage {
}
}
+ private final ExecutorService executorService;
+
+ @Inject
+ @VisibleForTesting
+ AwsS3ObjectStorage() {
+ executorService = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
+ }
+
+ @PreDestroy
+ public void tearDown() {
+ executorService.shutdownNow();
+ }
+
public static ObjectStorageBlobsDAOBuilder.RequireContainerName daoBuilder(AwsS3AuthConfiguration configuration) {
return ObjectStorageBlobsDAOBuilder.forBlobStore(new BlobStoreBuilder(configuration));
}
- public static Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) {
+ public Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) {
return Optional.of((blob) -> {
File file = null;
try {
@@ -94,7 +110,7 @@ public class AwsS3ObjectStorage {
});
}
- private static void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
+ private void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
try {
put(containerName, configuration, blob, file);
} catch (RuntimeException e) {
@@ -106,7 +122,7 @@ public class AwsS3ObjectStorage {
}
}
- private static void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
+ private void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
try {
PutObjectRequest request = new PutObjectRequest(containerName.value(),
blob.getMetadata().getName(),
@@ -120,7 +136,7 @@ public class AwsS3ObjectStorage {
}
}
- private static TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
+ private TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
ClientConfiguration clientConfiguration = getClientConfiguration();
AmazonS3 amazonS3 = getS3Client(configuration, clientConfiguration);
@@ -128,7 +144,7 @@ public class AwsS3ObjectStorage {
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold(MULTIPART_UPLOAD_THRESHOLD.getValue())
- .withExecutorFactory(() -> EXECUTOR_SERVICE)
+ .withExecutorFactory(() -> executorService)
.withShutDownThreadPools(DO_NOT_SHUTDOWN_THREAD_POOL)
.build();
}
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
index 6a0367e..3e71477 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
@@ -28,6 +28,7 @@ import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAO;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOContract;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -37,9 +38,11 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
private ContainerName containerName;
private AwsS3AuthConfiguration configuration;
+ private AwsS3ObjectStorage awsS3ObjectStorage;
@BeforeEach
void setUp(DockerAwsS3Container dockerAwsS3Container) {
+ awsS3ObjectStorage = new AwsS3ObjectStorage();
containerName = ContainerName.of(UUID.randomUUID().toString());
configuration = AwsS3AuthConfiguration.builder()
.endpoint(dockerAwsS3Container.getEndpoint())
@@ -48,6 +51,11 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
.build();
}
+ @AfterEach
+ void tearDown() {
+ awsS3ObjectStorage.tearDown();
+ }
+
@Override
public ContainerName containerName() {
return containerName;
@@ -79,7 +87,7 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
.builder(configuration)
.container(containerName)
.blobIdFactory(new HashBlobId.Factory())
- .putBlob(AwsS3ObjectStorage.putBlob(containerName, configuration));
+ .putBlob(awsS3ObjectStorage.putBlob(containerName, configuration));
assertBlobsDAOCanStoreAndRetrieve(builder);
}
diff --git a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
index 5e43388..a2b3953 100644
--- a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
+++ b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
+import javax.inject.Provider;
import javax.inject.Singleton;
import org.apache.commons.configuration.Configuration;
@@ -64,12 +65,12 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
@Provides
@Singleton
- private ObjectStorageBlobsDAO buildObjectStore(ObjectStorageBlobConfiguration configuration, BlobId.Factory blobIdFactory) throws InterruptedException, ExecutionException, TimeoutException {
+ private ObjectStorageBlobsDAO buildObjectStore(ObjectStorageBlobConfiguration configuration, BlobId.Factory blobIdFactory, Provider<AwsS3ObjectStorage> awsS3ObjectStorageProvider) throws InterruptedException, ExecutionException, TimeoutException {
ObjectStorageBlobsDAO dao = selectDaoBuilder(configuration)
.container(configuration.getNamespace())
.blobIdFactory(blobIdFactory)
.payloadCodec(configuration.getPayloadCodec())
- .putBlob(putBlob(blobIdFactory, configuration))
+ .putBlob(putBlob(blobIdFactory, configuration, awsS3ObjectStorageProvider))
.build();
dao.createContainer(configuration.getNamespace()).block(Duration.ofMinutes(1));
return dao;
@@ -85,12 +86,14 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
}
- private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration) {
+ private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration, Provider<AwsS3ObjectStorage> awsS3ObjectStorageProvider) {
switch (configuration.getProvider()) {
case SWIFT:
return Optional.empty();
case AWSS3:
- return AwsS3ObjectStorage.putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
+ return awsS3ObjectStorageProvider
+ .get()
+ .putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
}
throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 05/07: JAMES-2725 remove BlobIdFactory parameter
from AwsS3ObjectStorage.putBlob
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit ad14321a2e5dd1df3f554cf65d759c69e9c679b4
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Tue May 7 14:57:16 2019 +0200
JAMES-2725 remove BlobIdFactory parameter from AwsS3ObjectStorage.putBlob
---
.../james/blob/objectstorage/aws/AwsS3ObjectStorage.java | 13 ++++++-------
.../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 3 +--
.../objectstorage/ObjectStorageDependenciesModule.java | 2 +-
3 files changed, 8 insertions(+), 10 deletions(-)
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index f00be8b..fbf48a4 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -29,7 +29,6 @@ import java.util.concurrent.Executors;
import java.util.function.Supplier;
import org.apache.commons.io.FileUtils;
-import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
import org.apache.james.blob.objectstorage.PutBlobFunction;
@@ -78,13 +77,13 @@ public class AwsS3ObjectStorage {
return ObjectStorageBlobsDAOBuilder.forBlobStore(new BlobStoreBuilder(configuration));
}
- public static Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration) {
+ public static Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) {
return Optional.of((blob) -> {
File file = null;
try {
file = File.createTempFile(UUID.randomUUID().toString(), ".tmp");
FileUtils.copyToFile(blob.getPayload().openStream(), file);
- putWithRetry(blobIdFactory, containerName, configuration, blob, file, FIRST_TRY);
+ putWithRetry(containerName, configuration, blob, file, FIRST_TRY);
} catch (IOException e) {
throw new RuntimeException(e);
} finally {
@@ -95,19 +94,19 @@ public class AwsS3ObjectStorage {
});
}
- private static void putWithRetry(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
+ private static void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
try {
- put(blobIdFactory, containerName, configuration, blob, file);
+ put(containerName, configuration, blob, file);
} catch (RuntimeException e) {
if (tried < MAX_RETRY_ON_EXCEPTION) {
- putWithRetry(blobIdFactory, containerName, configuration, blob, file, tried + 1);
+ putWithRetry(containerName, configuration, blob, file, tried + 1);
} else {
throw e;
}
}
}
- private static void put(BlobId.Factory blobIdFactory, ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
+ private static void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
try {
PutObjectRequest request = new PutObjectRequest(containerName.value(),
blob.getMetadata().getName(),
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
index 9ecda9f..6a0367e 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
@@ -24,7 +24,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.UUID;
import org.apache.james.blob.api.HashBlobId;
-import org.apache.james.blob.api.TestBlobId;
import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAO;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
@@ -80,7 +79,7 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
.builder(configuration)
.container(containerName)
.blobIdFactory(new HashBlobId.Factory())
- .putBlob(AwsS3ObjectStorage.putBlob(new TestBlobId.Factory(), containerName, configuration));
+ .putBlob(AwsS3ObjectStorage.putBlob(containerName, configuration));
assertBlobsDAOCanStoreAndRetrieve(builder);
}
diff --git a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
index 23c40b9..5e43388 100644
--- a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
+++ b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
@@ -90,7 +90,7 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
case SWIFT:
return Optional.empty();
case AWSS3:
- return AwsS3ObjectStorage.putBlob(blobIdFactory, configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
+ return AwsS3ObjectStorage.putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
}
throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 07/07: Merge remote-tracking branch
'remk/JAMES-2725-2'
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit db3aad06dc561e8ee5300b32d6b2f2045812e057
Merge: 2a91b93 712f005
Author: Raphael Ouazana <ra...@linagora.com>
AuthorDate: Mon May 13 14:37:15 2019 +0200
Merge remote-tracking branch 'remk/JAMES-2725-2'
.../java/org/apache/james/blob/api/BlobStore.java | 2 +-
.../apache/james/blob/api/MetricableBlobStore.java | 4 +-
.../main/java/org/apache/james/blob/api/Store.java | 28 +---
.../apache/james/blob/api/BlobStoreContract.java | 6 +-
.../james/blob/api/FixedLengthInputStreamTest.java | 63 -------
.../blob/api/MetricableBlobStoreContract.java | 4 +-
.../james/blob/cassandra/CassandraBlobsDAO.java | 2 +-
.../blob/cassandra/CassandraBlobsDAOTest.java | 2 +-
.../apache/james/blob/memory/MemoryBlobStore.java | 2 +-
server/blob/blob-objectstorage/pom.xml | 5 +
.../blob/objectstorage/ObjectStorageBlobsDAO.java | 17 +-
.../ObjectStorageBlobsDAOBuilder.java | 19 ++-
.../james/blob/objectstorage/PutBlobFunction.java | 37 +++++
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 128 +++++++++++++-
.../objectstorage/ObjectStorageBlobsDAOTest.java | 2 +-
.../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 11 +-
.../apache/james/blob/union/UnionBlobStore.java | 8 +-
.../james/blob/union/UnionBlobStoreTest.java | 14 +-
.../apache/james/blob/mail/MimeMessageStore.java | 6 +-
.../ObjectStorageDependenciesModule.java | 19 ++-
.../james/CassandraRabbitMQJamesServerFixture.java | 43 +++++
.../james/CassandraRabbitMQJamesServerTest.java | 184 ---------------------
.../apache/james/MailsShouldBeWellReceived.java | 66 ++++++++
.../org/apache/james/WithCassandraBlobStore.java | 68 ++++++++
.../apache/james/WithCassandraBlobStoreTest.java | 26 +++
.../apache/james/WithDefaultAwsS3Extension.java | 71 ++++++++
.../org/apache/james/WithDefaultAwsS3Test.java | 41 +++++
.../apache/james/WithDefaultSwiftExtension.java | 71 ++++++++
.../org/apache/james/WithDefaultSwiftTest.java | 41 +++++
.../apache/james/WithEncryptedAwsS3Extension.java | 72 ++++++++
.../org/apache/james/WithEncryptedAwsS3Test.java | 41 +++++
.../apache/james/WithEncryptedSwiftExtension.java | 72 ++++++++
.../org/apache/james/WithEncryptedSwiftTest.java | 42 +++++
.../org/apache/james/JamesServerExtension.java | 5 +
.../james/webadmin/vault/routes/ExportService.java | 2 +-
35 files changed, 913 insertions(+), 311 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 03/07: JAMES-2725 add retry policy for s3 upload
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit e4832917d9d266cd4c77aae9bb3d8c1105baff7e
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Tue May 7 11:05:11 2019 +0200
JAMES-2725 add retry policy for s3 upload
---
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 28 +++++++++++++++++-----
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index 7940ef0..56d58a3 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -42,9 +42,11 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import com.amazonaws.AmazonClientException;
+import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.retry.PredefinedRetryPolicies;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.PutObjectRequest;
@@ -59,7 +61,9 @@ public class AwsS3ObjectStorage {
public static final int MAX_THREADS = 5;
private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false;
- private static Size MULTIPART_UPLOAD_THRESHOLD;
+ public static final int MAX_UPLOAD_THREADS = 5;
+ private static final int MAX_ERROR_RETRY = 5;
+ public static Size MULTIPART_UPLOAD_THRESHOLD;
static {
try {
@@ -106,11 +110,8 @@ public class AwsS3ObjectStorage {
}
private static TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
- AmazonS3 amazonS3 = AmazonS3ClientBuilder
- .standard()
- .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(configuration.getAccessKeyId(), configuration.getSecretKey())))
- .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(configuration.getEndpoint(), null))
- .build();
+ ClientConfiguration clientConfiguration = getClientConfiguration();
+ AmazonS3 amazonS3 = getS3Client(configuration, clientConfiguration);
return TransferManagerBuilder
.standard()
@@ -121,6 +122,21 @@ public class AwsS3ObjectStorage {
.build();
}
+ private static AmazonS3 getS3Client(AwsS3AuthConfiguration configuration, ClientConfiguration clientConfiguration) {
+ return AmazonS3ClientBuilder
+ .standard()
+ .withClientConfiguration(clientConfiguration)
+ .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(configuration.getAccessKeyId(), configuration.getSecretKey())))
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(configuration.getEndpoint(), null))
+ .build();
+ }
+
+ private static ClientConfiguration getClientConfiguration() {
+ ClientConfiguration clientConfiguration = new ClientConfiguration();
+ clientConfiguration.setRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(MAX_ERROR_RETRY));
+ return clientConfiguration;
+ }
+
private static class BlobStoreBuilder implements Supplier<BlobStore> {
private final AwsS3AuthConfiguration configuration;
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org