You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2020/09/15 10:44:40 UTC
[james-project] 14/25: JAMES-3028 Expose DeDuplication/PassThrough
choice for S3 DumbBlobStore
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit d5e93159b9f72aa9e90636efcd9811344fb9489d
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Fri Jul 31 14:27:19 2020 +0700
JAMES-3028 Expose DeDuplication/PassThrough choice for S3 DumbBlobStore
---
.../CassandraRabbitMQAwsS3SmtpTestRuleFactory.java | 8 +-
server/blob/blob-s3/pom.xml | 5 +
.../james/blob/objectstorage/aws/S3BlobStore.java | 130 ---------------------
...Test.java => S3DeDuplicationBlobStoreTest.java} | 12 +-
...reTest.java => S3PassThroughBlobStoreTest.java} | 12 +-
.../modules/blobstore/BlobStoreModulesChooser.java | 18 +--
6 files changed, 30 insertions(+), 155 deletions(-)
diff --git a/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java b/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java
index d97bbd7..a09f56e 100644
--- a/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java
+++ b/mpt/impl/smtp/cassandra-rabbitmq-object-storage/src/test/java/org/apache/james/mpt/smtp/CassandraRabbitMQAwsS3SmtpTestRuleFactory.java
@@ -26,9 +26,9 @@ import org.apache.james.backends.cassandra.DockerCassandra;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import org.apache.james.backends.rabbitmq.DockerRabbitMQSingleton;
import org.apache.james.blob.api.BlobStore;
-import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DumbBlobStore;
import org.apache.james.blob.api.MetricableBlobStore;
-import org.apache.james.blob.objectstorage.aws.S3BlobStore;
+import org.apache.james.blob.objectstorage.aws.S3DumbBlobStore;
import org.apache.james.dnsservice.api.DNSService;
import org.apache.james.modules.TestRabbitMQModule;
import org.apache.james.modules.mailbox.KeyspacesConfiguration;
@@ -38,6 +38,7 @@ import org.apache.james.modules.rabbitmq.RabbitMQModule;
import org.apache.james.modules.server.CamelMailetContainerModule;
import org.apache.james.queue.api.MailQueueItemDecoratorFactory;
import org.apache.james.queue.api.RawMailQueueItemDecoratorFactory;
+import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore;
import org.apache.james.server.core.configuration.Configuration;
import org.apache.james.util.Host;
import org.junit.rules.TemporaryFolder;
@@ -58,7 +59,8 @@ public final class CassandraRabbitMQAwsS3SmtpTestRuleFactory {
protected void configure() {
bind(BlobStore.class)
.annotatedWith(Names.named(MetricableBlobStore.BLOB_STORE_IMPLEMENTATION))
- .to(S3BlobStore.class);
+ .to(DeDuplicationBlobStore.class);
+ bind(DumbBlobStore.class).to(S3DumbBlobStore.class);
}
};
diff --git a/server/blob/blob-s3/pom.xml b/server/blob/blob-s3/pom.xml
index b57a33b..31cf300 100644
--- a/server/blob/blob-s3/pom.xml
+++ b/server/blob/blob-s3/pom.xml
@@ -49,6 +49,11 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-storage-strategy</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>james-core</artifactId>
</dependency>
<dependency>
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStore.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStore.java
deleted file mode 100644
index 1eba115..0000000
--- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStore.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/****************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one *
- * or more contributor license agreements. See the NOTICE file *
- * distributed with this work for additional information *
- * regarding copyright ownership. The ASF licenses this file *
- * to you under the Apache License, Version 2.0 (the *
- * "License"); you may not use this file except in compliance *
- * with the License. You may obtain a copy of the License at *
- * *
- * http://www.apache.org/licenses/LICENSE-2.0 *
- * *
- * Unless required by applicable law or agreed to in writing, *
- * software distributed under the License is distributed on an *
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
- * KIND, either express or implied. See the License for the *
- * specific language governing permissions and limitations *
- * under the License. *
- ****************************************************************/
-
-package org.apache.james.blob.objectstorage.aws;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-import javax.inject.Inject;
-
-import org.apache.commons.io.IOUtils;
-import org.apache.james.blob.api.BlobId;
-import org.apache.james.blob.api.BlobStore;
-import org.apache.james.blob.api.BucketName;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.hash.Hashing;
-import com.google.common.hash.HashingInputStream;
-import com.google.common.io.FileBackedOutputStream;
-
-import reactor.core.publisher.Mono;
-import reactor.util.function.Tuples;
-
-public class S3BlobStore implements BlobStore {
-
- private static final boolean LAZY_RESOURCE_CLEANUP = false;
- private static final int FILE_THRESHOLD = 1024 * 100;
- private final S3DumbBlobStore dumbBlobStore;
- private final BlobId.Factory blobIdFactory;
- private final BucketName defaultBucket;
-
- @Inject
- @VisibleForTesting
- S3BlobStore(S3DumbBlobStore dumbBlobStore, BlobId.Factory blobIdFactory, BucketName defaultBucket) {
- this.dumbBlobStore = dumbBlobStore;
- this.blobIdFactory = blobIdFactory;
- this.defaultBucket = defaultBucket;
- }
-
- @Override
- public Mono<BlobId> save(BucketName bucketName, byte[] data, StoragePolicy storagePolicy) {
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(data);
-
- BlobId blobId = blobIdFactory.forPayload(data);
-
- return dumbBlobStore.save(bucketName, blobId, data)
- .then(Mono.just(blobId));
- }
-
- @Override
- public Mono<BlobId> save(BucketName bucketName, InputStream data, StoragePolicy storagePolicy) {
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(data);
- HashingInputStream hashingInputStream = new HashingInputStream(Hashing.sha256(), data);
- return Mono.using(
- () -> new FileBackedOutputStream(FILE_THRESHOLD),
- fileBackedOutputStream -> saveAndGenerateBlobId(bucketName, hashingInputStream, fileBackedOutputStream),
- fileBackedOutputStream -> {
- try {
- fileBackedOutputStream.reset();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- },
- LAZY_RESOURCE_CLEANUP);
- }
-
- private Mono<BlobId> saveAndGenerateBlobId(BucketName bucketName, HashingInputStream hashingInputStream, FileBackedOutputStream fileBackedOutputStream) {
- return Mono
- .fromCallable(() -> {
- IOUtils.copy(hashingInputStream, fileBackedOutputStream);
- return Tuples.of(blobIdFactory.from(hashingInputStream.hash().toString()), fileBackedOutputStream.asByteSource());
- })
- .flatMap(tuple -> dumbBlobStore.save(bucketName, tuple.getT1(), tuple.getT2()).thenReturn(tuple.getT1()));
- }
-
-
- @Override
- public Mono<byte[]> readBytes(BucketName bucketName, BlobId blobId) {
- Preconditions.checkNotNull(bucketName);
-
- return dumbBlobStore.readBytes(bucketName, blobId);
- }
-
- @Override
- public InputStream read(BucketName bucketName, BlobId blobId) {
- Preconditions.checkNotNull(bucketName);
-
- return dumbBlobStore.read(bucketName, blobId);
- }
-
- @Override
- public Mono<Void> deleteBucket(BucketName bucketName) {
- Preconditions.checkNotNull(bucketName);
-
- return dumbBlobStore.deleteBucket(bucketName);
- }
-
- @Override
- public BucketName getDefaultBucketName() {
- return defaultBucket;
- }
-
- @Override
- public Mono<Void> delete(BucketName bucketName, BlobId blobId) {
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(blobId);
-
- return dumbBlobStore.delete(bucketName, blobId);
- }
-
-}
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3DeDuplicationBlobStoreTest.java
similarity index 87%
copy from server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java
copy to server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3DeDuplicationBlobStoreTest.java
index 286c300..6bc1d34 100644
--- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3DeDuplicationBlobStoreTest.java
@@ -22,17 +22,17 @@ package org.apache.james.blob.objectstorage.aws;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BlobStoreContract;
-import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-class S3BlobStoreTest implements BlobStoreContract {
+class S3DeDuplicationBlobStoreTest implements BlobStoreContract {
- private static S3BlobStore testee;
+ private static BlobStore testee;
private static S3DumbBlobStore s3DumbBlobStore;
@BeforeAll
@@ -44,7 +44,11 @@ class S3BlobStoreTest implements BlobStoreContract {
.build();
s3DumbBlobStore = new S3DumbBlobStore(configuration, dockerAwsS3.dockerAwsS3().region());
- testee = new S3BlobStore(s3DumbBlobStore, new HashBlobId.Factory(), BucketName.DEFAULT);
+ testee = BlobStoreFactory.builder()
+ .dumbBlobStore(s3DumbBlobStore)
+ .blobIdFactory(new HashBlobId.Factory())
+ .defaultBucketName()
+ .passthrough();
}
@AfterEach
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3PassThroughBlobStoreTest.java
similarity index 87%
rename from server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java
rename to server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3PassThroughBlobStoreTest.java
index 286c300..71027d9 100644
--- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreTest.java
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3PassThroughBlobStoreTest.java
@@ -22,17 +22,17 @@ package org.apache.james.blob.objectstorage.aws;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BlobStoreContract;
-import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-class S3BlobStoreTest implements BlobStoreContract {
+class S3PassThroughBlobStoreTest implements BlobStoreContract {
- private static S3BlobStore testee;
+ private static BlobStore testee;
private static S3DumbBlobStore s3DumbBlobStore;
@BeforeAll
@@ -44,7 +44,11 @@ class S3BlobStoreTest implements BlobStoreContract {
.build();
s3DumbBlobStore = new S3DumbBlobStore(configuration, dockerAwsS3.dockerAwsS3().region());
- testee = new S3BlobStore(s3DumbBlobStore, new HashBlobId.Factory(), BucketName.DEFAULT);
+ testee = BlobStoreFactory.builder()
+ .dumbBlobStore(s3DumbBlobStore)
+ .blobIdFactory(new HashBlobId.Factory())
+ .defaultBucketName()
+ .deduplication();
}
@AfterEach
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
index 770cffa..0aab073 100644
--- a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
@@ -25,7 +25,6 @@ import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.DumbBlobStore;
import org.apache.james.blob.cassandra.CassandraDumbBlobStore;
import org.apache.james.blob.cassandra.cache.CachedBlobStore;
-import org.apache.james.blob.objectstorage.aws.S3BlobStore;
import org.apache.james.blob.objectstorage.aws.S3DumbBlobStore;
import org.apache.james.eventsourcing.Event;
import org.apache.james.eventsourcing.eventstore.cassandra.dto.EventDTO;
@@ -64,10 +63,6 @@ public class BlobStoreModulesChooser {
install(new S3BlobStoreModule());
bind(DumbBlobStore.class).to(S3DumbBlobStore.class);
-
- bind(BlobStore.class)
- .annotatedWith(Names.named(CachedBlobStore.BACKEND))
- .to(S3BlobStore.class);
}
}
@@ -94,16 +89,11 @@ public class BlobStoreModulesChooser {
@VisibleForTesting
public static List<Module> chooseModules(BlobStoreConfiguration choosingConfiguration) {
- ImmutableList.Builder<Module> moduleBuilder = ImmutableList.<Module>builder()
+ return ImmutableList.<Module>builder()
.add(chooseDumBlobStoreModule(choosingConfiguration.getImplementation()))
- .add(new StoragePolicyConfigurationSanityEnforcementModule(choosingConfiguration));
-
- //TODO JAMES-3028 add the storage policy module for all implementation and unbind the ObjectStorageBlobStore
- if (choosingConfiguration.getImplementation() == BlobStoreConfiguration.BlobStoreImplName.CASSANDRA) {
- moduleBuilder.add(
- chooseStoragePolicyModule(choosingConfiguration.storageStrategy()));
- }
- return moduleBuilder.build();
+ .add(chooseStoragePolicyModule(choosingConfiguration.storageStrategy()))
+ .add(new StoragePolicyConfigurationSanityEnforcementModule(choosingConfiguration))
+ .build();
}
public static Module chooseDumBlobStoreModule(BlobStoreConfiguration.BlobStoreImplName implementation) {
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org