You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by ro...@apache.org on 2019/05/13 12:38:12 UTC
[james-project] 03/07: JAMES-2725 add retry policy for s3 upload
This is an automated email from the ASF dual-hosted git repository.
rouazana pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit e4832917d9d266cd4c77aae9bb3d8c1105baff7e
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Tue May 7 11:05:11 2019 +0200
JAMES-2725 add retry policy for s3 upload
---
.../blob/objectstorage/aws/AwsS3ObjectStorage.java | 28 +++++++++++++++++-----
1 file changed, 22 insertions(+), 6 deletions(-)
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
index 7940ef0..56d58a3 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/aws/AwsS3ObjectStorage.java
@@ -42,9 +42,11 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.logging.slf4j.config.SLF4JLoggingModule;
import com.amazonaws.AmazonClientException;
+import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
+import com.amazonaws.retry.PredefinedRetryPolicies;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.PutObjectRequest;
@@ -59,7 +61,9 @@ public class AwsS3ObjectStorage {
public static final int MAX_THREADS = 5;
private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(MAX_THREADS, NamedThreadFactory.withClassName(AwsS3ObjectStorage.class));
private static final boolean DO_NOT_SHUTDOWN_THREAD_POOL = false;
- private static Size MULTIPART_UPLOAD_THRESHOLD;
+ public static final int MAX_UPLOAD_THREADS = 5;
+ private static final int MAX_ERROR_RETRY = 5;
+ public static Size MULTIPART_UPLOAD_THRESHOLD;
static {
try {
@@ -106,11 +110,8 @@ public class AwsS3ObjectStorage {
}
private static TransferManager getTransferManager(AwsS3AuthConfiguration configuration) {
- AmazonS3 amazonS3 = AmazonS3ClientBuilder
- .standard()
- .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(configuration.getAccessKeyId(), configuration.getSecretKey())))
- .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(configuration.getEndpoint(), null))
- .build();
+ ClientConfiguration clientConfiguration = getClientConfiguration();
+ AmazonS3 amazonS3 = getS3Client(configuration, clientConfiguration);
return TransferManagerBuilder
.standard()
@@ -121,6 +122,21 @@ public class AwsS3ObjectStorage {
.build();
}
+ private static AmazonS3 getS3Client(AwsS3AuthConfiguration configuration, ClientConfiguration clientConfiguration) {
+ return AmazonS3ClientBuilder
+ .standard()
+ .withClientConfiguration(clientConfiguration)
+ .withCredentials(new AWSStaticCredentialsProvider(new BasicAWSCredentials(configuration.getAccessKeyId(), configuration.getSecretKey())))
+ .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(configuration.getEndpoint(), null))
+ .build();
+ }
+
+ private static ClientConfiguration getClientConfiguration() {
+ ClientConfiguration clientConfiguration = new ClientConfiguration();
+ clientConfiguration.setRetryPolicy(PredefinedRetryPolicies.getDefaultRetryPolicyWithCustomMaxRetries(MAX_ERROR_RETRY));
+ return clientConfiguration;
+ }
+
private static class BlobStoreBuilder implements Supplier<BlobStore> {
private final AwsS3AuthConfiguration configuration;
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org