You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by ro...@apache.org on 2019/05/13 12:38:15 UTC
[james-project] 06/07: JAMES-2725 Use injection for AwsS3Object
storage for managing its thread pool lifecycle.
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 712f005be906c4eefdf3158960407b4a58dc5357
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Thu May 9 18:02:48 2019 +0200
JAMES-2725 Use injection for AwsS3Object storage for managing its thread pool lifecycle.
---
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 30 +++++++++++++++++-----
.../aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java | 10 +++++++-
.../ObjectStorageDependenciesModule.java | 11 +++++---
3 files changed, 39 insertions(+), 12 deletions(-)
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index fbf48a4..5454af0 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -28,6 +28,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Supplier;
+import javax.annotation.PreDestroy;
+import javax.inject.Inject;
+
import org.apache.commons.io.FileUtils;
import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
@@ -51,14 +54,14 @@ import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
public class AwsS3ObjectStorage {
private static final Iterable<Module> JCLOUDS_MODULES = ImmutableSet.of(new SLF4JLoggingModule());
- public static final int MAX_THREADS = 5;
- private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
+ public static final int MAX_THREADS = 5;
private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false;
private static final int MAX_ERROR_RETRY = 5;
private static final int FIRST_TRY = 0;
@@ -73,11 +76,24 @@ public class AwsS3ObjectStorage {
}
}
+ private final ExecutorService executorService;
+
+ @Inject
+ @VisibleForTesting
+ AwsS3ObjectStorage() {
+ executorService = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
+ }
+
+ @PreDestroy
+ public void tearDown() {
+ executorService.shutdownNow();
+ }
+
public static ObjectStorageBlobsDAOBuilder.RequireContainerName daoBuilder(AwsS3AuthConfiguration configuration) {
return ObjectStorageBlobsDAOBuilder.forBlobStore(new BlobStoreBuilder(configuration));
}
- public static Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) {
+ public Optional<PutBlobFunction> putBlob(ContainerName containerName, AwsS3AuthConfiguration configuration) {
return Optional.of((blob) -> {
File file = null;
try {
@@ -94,7 +110,7 @@ public class AwsS3ObjectStorage {
});
}
- private static void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
+ private void putWithRetry(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file, int tried) {
try {
put(containerName, configuration, blob, file);
} catch (RuntimeException e) {
@@ -106,7 +122,7 @@ public class AwsS3ObjectStorage {
}
}
- private static void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
+ private void put(ContainerName containerName, AwsS3AuthConfiguration configuration, Blob blob, File file) {
try {
PutObjectRequest request = new PutObjectRequest(containerName.value(),
blob.getMetadata().getName(),
@@ -120,7 +136,7 @@ public class AwsS3ObjectStorage {
}
}
- private static TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
+ private TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
ClientConfiguration clientConfiguration = getClientConfiguration();
AmazonS3 amazonS3 = getS3Client(configuration, clientConfiguration);
@@ -128,7 +144,7 @@ public class AwsS3ObjectStorage {
.standard()
.withS3Client(amazonS3)
.withMultipartUploadThreshold(MULTIPART_UPLOAD_THRESHOLD.getValue())
- .withExecutorFactory(() -> EXECUTOR_SERVICE)
+ .withExecutorFactory(() -> executorService)
.withShutDownThreadPools(DO_NOT_SHUTDOWN_THREAD_POOL)
.build();
}
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
index 6a0367e..3e71477 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorageBlobsDAOBuilderTest.java
@@ -28,6 +28,7 @@ import org.apache.james.blob.objectstorage.ContainerName;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAO;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOBuilder;
import org.apache.james.blob.objectstorage.ObjectStorageBlobsDAOContract;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -37,9 +38,11 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
private ContainerName containerName;
private AwsS3AuthConfiguration configuration;
+ private AwsS3ObjectStorage awsS3ObjectStorage;
@BeforeEach
void setUp(DockerAwsS3Container dockerAwsS3Container) {
+ awsS3ObjectStorage = new AwsS3ObjectStorage();
containerName = ContainerName.of(UUID.randomUUID().toString());
configuration = AwsS3AuthConfiguration.builder()
.endpoint(dockerAwsS3Container.getEndpoint())
@@ -48,6 +51,11 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
.build();
}
+ @AfterEach
+ void tearDown() {
+ awsS3ObjectStorage.tearDown();
+ }
+
@Override
public ContainerName containerName() {
return containerName;
@@ -79,7 +87,7 @@ class AwsS3ObjectStorageBlobsDAOBuilderTest implements ObjectStorageBlobsDAOCont
.builder(configuration)
.container(containerName)
.blobIdFactory(new HashBlobId.Factory())
- .putBlob(AwsS3ObjectStorage.putBlob(containerName, configuration));
+ .putBlob(awsS3ObjectStorage.putBlob(containerName, configuration));
assertBlobsDAOCanStoreAndRetrieve(builder);
}
diff --git a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
index 5e43388..a2b3953 100644
--- a/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
+++ b/server/container/guice/blob-objectstorage-guice/src/main/java/org/apache/james/modules/objectstorage/ObjectStorageDependenciesModule.java
@@ -25,6 +25,7 @@ import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
+import javax.inject.Provider;
import javax.inject.Singleton;
import org.apache.commons.configuration.Configuration;
@@ -64,12 +65,12 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
@Provides
@Singleton
- private ObjectStorageBlobsDAO buildObjectStore(ObjectStorageBlobConfiguration configuration, BlobId.Factory blobIdFactory) throws InterruptedException, ExecutionException, TimeoutException {
+ private ObjectStorageBlobsDAO buildObjectStore(ObjectStorageBlobConfiguration configuration, BlobId.Factory blobIdFactory, Provider<AwsS3ObjectStorage> awsS3ObjectStorageProvider) throws InterruptedException, ExecutionException, TimeoutException {
ObjectStorageBlobsDAO dao = selectDaoBuilder(configuration)
.container(configuration.getNamespace())
.blobIdFactory(blobIdFactory)
.payloadCodec(configuration.getPayloadCodec())
- .putBlob(putBlob(blobIdFactory, configuration))
+ .putBlob(putBlob(blobIdFactory, configuration, awsS3ObjectStorageProvider))
.build();
dao.createContainer(configuration.getNamespace()).block(Duration.ofMinutes(1));
return dao;
@@ -85,12 +86,14 @@ public class ObjectStorageDependenciesModule extends AbstractModule {
throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
}
- private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration) {
+ private Optional<PutBlobFunction> putBlob(BlobId.Factory blobIdFactory, ObjectStorageBlobConfiguration configuration, Provider<AwsS3ObjectStorage> awsS3ObjectStorageProvider) {
switch (configuration.getProvider()) {
case SWIFT:
return Optional.empty();
case AWSS3:
- return AwsS3ObjectStorage.putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
+ return awsS3ObjectStorageProvider
+ .get()
+ .putBlob(configuration.getNamespace(), (AwsS3AuthConfiguration) configuration.getSpecificAuthConfiguration());
}
throw new IllegalArgumentException("unknown provider " + configuration.getProvider());
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org