You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2019/07/08 15:59:38 UTC
[james-project] 02/07: JAMES-2806 ObjectStorage Swift save retry
save() on non existing bucket
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 542b4cbf4fdc7aac7c5d59e8879fea3063e2c963
Author: Tran Tien Duc <dt...@linagora.com>
AuthorDate: Thu Jul 4 17:09:03 2019 +0700
JAMES-2806 ObjectStorage Swift save retry save() on non existing bucket
---
server/blob/blob-objectstorage/pom.xml | 4 +++
.../objectstorage/StreamCompatibleBlobPutter.java | 34 ++++++++++++++++++++--
.../objectstorage/ObjectStorageBlobsDAOTest.java | 12 +++-----
3 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/server/blob/blob-objectstorage/pom.xml b/server/blob/blob-objectstorage/pom.xml
index c312e93..dabae94 100644
--- a/server/blob/blob-objectstorage/pom.xml
+++ b/server/blob/blob-objectstorage/pom.xml
@@ -87,6 +87,10 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
+ <groupId>io.projectreactor.addons</groupId>
+ <artifactId>reactor-extra</artifactId>
+ </dependency>
+ <dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<scope>test</scope>
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java
index f01bc34..1bac8e2 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/StreamCompatibleBlobPutter.java
@@ -19,6 +19,7 @@
package org.apache.james.blob.objectstorage;
+import java.time.Duration;
import java.util.function.Supplier;
import org.apache.james.blob.api.BlobId;
@@ -26,9 +27,21 @@ import org.apache.james.blob.api.BucketName;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.options.CopyOptions;
+import org.jclouds.domain.Location;
+import org.jclouds.http.HttpResponseException;
+
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+import reactor.retry.Retry;
public class StreamCompatibleBlobPutter implements BlobPutter {
- private final org.jclouds.blobstore.BlobStore blobStore;
+
+ private static final int MAX_RETRIES = 3;
+ private static final Duration FIRST_BACK_OFF = Duration.ofMillis(100);
+ private static final Duration FOREVER = Duration.ofMillis(Long.MAX_VALUE);
+ private static final Location DEFAULT_LOCATION = null;
+
+ private final BlobStore blobStore;
public StreamCompatibleBlobPutter(BlobStore blobStore) {
this.blobStore = blobStore;
@@ -36,7 +49,14 @@ public class StreamCompatibleBlobPutter implements BlobPutter {
@Override
public void putDirectly(BucketName bucketName, Blob blob) {
- blobStore.putBlob(bucketName.asString(), blob);
+ Mono.fromRunnable(() -> blobStore.putBlob(bucketName.asString(), blob))
+ .publishOn(Schedulers.elastic())
+ .retryWhen(Retry.onlyIf(retryContext -> needToCreateBucket(retryContext.exception(), bucketName))
+ .exponentialBackoff(FIRST_BACK_OFF, FOREVER)
+ .withBackoffScheduler(Schedulers.elastic())
+ .retryMax(MAX_RETRIES)
+ .doOnRetry(retryContext -> blobStore.createContainerInLocation(DEFAULT_LOCATION, bucketName.asString())))
+ .block();
}
@Override
@@ -52,4 +72,14 @@ public class StreamCompatibleBlobPutter implements BlobPutter {
blobStore.copyBlob(bucketNameAsString, from, bucketNameAsString, to, CopyOptions.NONE);
blobStore.removeBlob(bucketNameAsString, from);
}
+
+ private boolean needToCreateBucket(Throwable throwable, BucketName bucketName) {
+ if (throwable instanceof HttpResponseException) {
+ HttpResponseException ex = (HttpResponseException) throwable;
+ return ex.getCommand().getCurrentRequest().getMethod().equals("PUT")
+ && !blobStore.containerExists(bucketName.asString());
+ }
+
+ return false;
+ }
}
\ No newline at end of file
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java
index 12f11b8..a6321f7 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobsDAOTest.java
@@ -31,6 +31,7 @@ import java.util.UUID;
import org.apache.commons.io.IOUtils;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.BucketBlobStoreContract;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
@@ -45,7 +46,6 @@ import org.apache.james.blob.objectstorage.swift.UserHeaderName;
import org.apache.james.blob.objectstorage.swift.UserName;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -56,7 +56,8 @@ import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
@ExtendWith(DockerSwiftExtension.class)
-public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract, BucketBlobStoreContract {
+
private static final String BIG_STRING = Strings.repeat("big blob content", 10 * 1024);
private static final TenantName TENANT_NAME = TenantName.of("test");
private static final UserName USER_NAME = UserName.of("tester");
@@ -98,6 +99,7 @@ public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract {
@AfterEach
void tearDown() {
blobStore.deleteContainer(defaultBucketName.asString());
+ blobStore.deleteContainer(CUSTOM.asString());
blobStore.getContext().close();
}
@@ -111,12 +113,6 @@ public class ObjectStorageBlobsDAOTest implements MetricableBlobStoreContract {
return new HashBlobId.Factory();
}
- @Override
- @Disabled("JAMES-2806: delete bucket not implemented yet for ObjectStorage")
- public void deleteBucketShouldPublishDeleteTimerMetrics() {
-
- }
-
@Test
void createBucketShouldMakeTheContainerToExist() {
BucketName bucketName = BucketName.of(UUID.randomUUID().toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org