You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by rc...@apache.org on 2020/07/28 03:20:40 UTC
[james-project] 12/25: JAMES-3318 BlobStoreDeletedMessageVault
should use DumbBlobStore for effective deletion
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 71187b503a5b5f012b590289363d0992ed067ca8
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Fri Jul 24 09:26:11 2020 +0700
JAMES-3318 BlobStoreDeletedMessageVault should use DumbBlobStore for effective deletion
---
.../vault/blob/BlobStoreDeletedMessageVault.java | 7 +-
.../james/vault/DeletedMessageVaultHookTest.java | 9 ++-
.../blob/BlobStoreDeletedMessageVaultTest.java | 9 ++-
.../blob/objectstorage/ObjectStorageBlobStore.java | 7 ++
.../modules/blobstore/BlobStoreModulesChooser.java | 1 +
.../james/modules/blobstore/NoopDumbBlobStore.java | 83 ++++++++++++++++++++++
.../routes/DeletedMessagesVaultRoutesTest.java | 9 ++-
7 files changed, 114 insertions(+), 11 deletions(-)
diff --git a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java
index 8febe09..2bc66b4 100644
--- a/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java
+++ b/mailbox/plugin/deleted-messages-vault/src/main/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVault.java
@@ -30,6 +30,7 @@ import javax.inject.Inject;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DumbBlobStore;
import org.apache.james.blob.api.ObjectNotFoundException;
import org.apache.james.core.Username;
import org.apache.james.mailbox.model.MessageId;
@@ -67,6 +68,7 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault {
private final MetricFactory metricFactory;
private final DeletedMessageMetadataVault messageMetadataVault;
private final BlobStore blobStore;
+ private final DumbBlobStore dumbBlobStore;
private final BucketNameGenerator nameGenerator;
private final Clock clock;
private final RetentionConfiguration retentionConfiguration;
@@ -74,12 +76,13 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault {
@Inject
public BlobStoreDeletedMessageVault(MetricFactory metricFactory, DeletedMessageMetadataVault messageMetadataVault,
- BlobStore blobStore, BucketNameGenerator nameGenerator,
+ BlobStore blobStore, DumbBlobStore dumbBlobStore, BucketNameGenerator nameGenerator,
Clock clock,
RetentionConfiguration retentionConfiguration) {
this.metricFactory = metricFactory;
this.messageMetadataVault = messageMetadataVault;
this.blobStore = blobStore;
+ this.dumbBlobStore = dumbBlobStore;
this.nameGenerator = nameGenerator;
this.clock = clock;
this.retentionConfiguration = retentionConfiguration;
@@ -156,7 +159,7 @@ public class BlobStoreDeletedMessageVault implements DeletedMessageVault {
return Mono.from(messageMetadataVault.retrieveStorageInformation(username, messageId))
.flatMap(storageInformation -> Mono.from(messageMetadataVault.remove(storageInformation.getBucketName(), username, messageId))
.thenReturn(storageInformation))
- .flatMap(storageInformation -> Mono.from(blobStore.delete(storageInformation.getBucketName(), storageInformation.getBlobId())))
+ .flatMap(storageInformation -> Mono.from(dumbBlobStore.delete(storageInformation.getBucketName(), storageInformation.getBlobId())))
.subscribeOn(Schedulers.elastic());
}
diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java
index 0fa94af..688574f 100644
--- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java
+++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java
@@ -30,7 +30,7 @@ import java.time.ZoneOffset;
import java.util.List;
import org.apache.james.blob.api.HashBlobId;
-import org.apache.james.blob.memory.MemoryBlobStoreFactory;
+import org.apache.james.blob.memory.MemoryDumbBlobStore;
import org.apache.james.core.MailAddress;
import org.apache.james.core.MaybeSender;
import org.apache.james.core.Username;
@@ -50,6 +50,7 @@ import org.apache.james.mailbox.model.MessageRange;
import org.apache.james.mailbox.model.SearchQuery;
import org.apache.james.metrics.tests.RecordingMetricFactory;
import org.apache.james.mime4j.dom.Message;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.apache.james.vault.blob.BlobStoreDeletedMessageVault;
import org.apache.james.vault.blob.BucketNameGenerator;
import org.apache.james.vault.memory.metadata.MemoryDeletedMessageMetadataVault;
@@ -108,11 +109,13 @@ class DeletedMessageVaultHookTest {
@BeforeEach
void setUp() throws Exception {
clock = Clock.fixed(DELETION_DATE.toInstant(), ZoneOffset.UTC);
+ MemoryDumbBlobStore dumbBlobStore = new MemoryDumbBlobStore();
messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(),
- MemoryBlobStoreFactory.builder()
+ BlobStoreFactory.builder()
+ .dumbBlobStore(dumbBlobStore)
.blobIdFactory(new HashBlobId.Factory())
.defaultBucketName()
- .passthrough(), new BucketNameGenerator(clock), clock,
+ .passthrough(), dumbBlobStore, new BucketNameGenerator(clock), clock,
RetentionConfiguration.DEFAULT);
DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter();
diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java
index 28b6506..982fefd 100644
--- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java
+++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java
@@ -40,8 +40,9 @@ import java.time.ZonedDateTime;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
-import org.apache.james.blob.memory.MemoryBlobStoreFactory;
+import org.apache.james.blob.memory.MemoryDumbBlobStore;
import org.apache.james.metrics.tests.RecordingMetricFactory;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.apache.james.utils.UpdatableTickingClock;
import org.apache.james.vault.DeletedMessageVault;
import org.apache.james.vault.DeletedMessageVaultContract;
@@ -63,12 +64,14 @@ class BlobStoreDeletedMessageVaultTest implements DeletedMessageVaultContract, D
void setUp() {
clock = new UpdatableTickingClock(NOW.toInstant());
metricFactory = new RecordingMetricFactory();
+ MemoryDumbBlobStore dumbBlobStore = new MemoryDumbBlobStore();
messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(),
- MemoryBlobStoreFactory.builder()
+ BlobStoreFactory.builder()
+ .dumbBlobStore(dumbBlobStore)
.blobIdFactory(new HashBlobId.Factory())
.defaultBucketName()
.passthrough(),
- new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT);
+ dumbBlobStore, new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT);
}
@Override
diff --git a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java
index 3c45fe8..cd7c7b3 100644
--- a/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java
+++ b/server/blob/blob-objectstorage/src/main/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStore.java
@@ -214,4 +214,11 @@ public class ObjectStorageBlobStore implements BlobStore {
return Mono.<Void>fromRunnable(() -> blobStore.removeBlob(resolvedBucketName.asString(), blobId.asString()))
.subscribeOn(Schedulers.elastic());
}
+
+ // Workaround while waiting for DumbBlobStore extraction
+ public Mono<Void> deleteEffectively(BucketName bucketName, BlobId blobId) {
+ ObjectStorageBucketName resolvedBucketName = bucketNameResolver.resolve(bucketName);
+ return Mono.<Void>fromRunnable(() -> blobStore.removeBlob(resolvedBucketName.asString(), blobId.asString()))
+ .subscribeOn(Schedulers.elastic());
+ }
}
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
index d790968..c0cdcc2 100644
--- a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
@@ -54,6 +54,7 @@ public class BlobStoreModulesChooser {
@Override
protected void configure() {
install(new ObjectStorageDependenciesModule());
+ bind(DumbBlobStore.class).to(NoopDumbBlobStore.class);
bind(BlobStore.class)
.annotatedWith(Names.named(CachedBlobStore.BACKEND))
.to(ObjectStorageBlobStore.class);
diff --git a/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/NoopDumbBlobStore.java b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/NoopDumbBlobStore.java
new file mode 100644
index 0000000..9ebc084
--- /dev/null
+++ b/server/container/guice/cassandra-rabbitmq-guice/src/main/java/org/apache/james/modules/blobstore/NoopDumbBlobStore.java
@@ -0,0 +1,83 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules.blobstore;
+
+import java.io.InputStream;
+
+import javax.inject.Inject;
+
+import org.apache.commons.lang3.NotImplementedException;
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DumbBlobStore;
+import org.apache.james.blob.api.ObjectNotFoundException;
+import org.apache.james.blob.api.ObjectStoreIOException;
+import org.apache.james.blob.objectstorage.ObjectStorageBlobStore;
+import org.reactivestreams.Publisher;
+
+import com.google.common.io.ByteSource;
+
+
+/**
+ * This class is a workaround while waiting for a real DumbBlobStore to be extracted from ObjectStorageBlobStore
+ */
+public class NoopDumbBlobStore implements DumbBlobStore {
+ private final ObjectStorageBlobStore objectStorageBlobStore;
+
+ @Inject
+ public NoopDumbBlobStore(ObjectStorageBlobStore objectStorageBlobStore) {
+ this.objectStorageBlobStore = objectStorageBlobStore;
+ }
+
+ @Override
+ public InputStream read(BucketName bucketName, BlobId blobId) throws ObjectStoreIOException, ObjectNotFoundException {
+ throw new NotImplementedException("Not implemented");
+ }
+
+ @Override
+ public Publisher<byte[]> readBytes(BucketName bucketName, BlobId blobId) {
+ throw new NotImplementedException("Not implemented");
+ }
+
+ @Override
+ public Publisher<Void> save(BucketName bucketName, BlobId blobId, byte[] data) {
+ throw new NotImplementedException("Not implemented");
+ }
+
+ @Override
+ public Publisher<Void> save(BucketName bucketName, BlobId blobId, InputStream inputStream) {
+ throw new NotImplementedException("Not implemented");
+ }
+
+ @Override
+ public Publisher<Void> save(BucketName bucketName, BlobId blobId, ByteSource content) {
+ throw new NotImplementedException("Not implemented");
+ }
+
+ @Override
+ public Publisher<Void> delete(BucketName bucketName, BlobId blobId) {
+ return objectStorageBlobStore.deleteEffectively(bucketName, blobId);
+ }
+
+ @Override
+ public Publisher<Void> deleteBucket(BucketName bucketName) {
+ throw new NotImplementedException("Not implemented");
+ }
+}
diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
index 1c8d463..cfdb6b3 100644
--- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
@@ -76,7 +76,7 @@ import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.export.api.BlobExportMechanism;
-import org.apache.james.blob.memory.MemoryBlobStoreFactory;
+import org.apache.james.blob.memory.MemoryDumbBlobStore;
import org.apache.james.core.Domain;
import org.apache.james.core.MailAddress;
import org.apache.james.core.MaybeSender;
@@ -102,6 +102,7 @@ import org.apache.james.mailbox.model.MessageResult;
import org.apache.james.mailbox.model.MultimailboxesSearchQuery;
import org.apache.james.mailbox.model.SearchQuery;
import org.apache.james.metrics.tests.RecordingMetricFactory;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.apache.james.task.Hostname;
import org.apache.james.task.MemoryTaskManager;
import org.apache.james.user.memory.MemoryUsersRepository;
@@ -181,13 +182,15 @@ class DeletedMessagesVaultRoutesTest {
@BeforeEach
void beforeEach() throws Exception {
blobIdFactory = new HashBlobId.Factory();
- blobStore = spy(MemoryBlobStoreFactory.builder()
+ MemoryDumbBlobStore dumbBlobStore = new MemoryDumbBlobStore();
+ blobStore = spy(BlobStoreFactory.builder()
+ .dumbBlobStore(dumbBlobStore)
.blobIdFactory(blobIdFactory)
.defaultBucketName()
.passthrough());
clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant());
vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(),
- blobStore, new BucketNameGenerator(clock), clock,
+ blobStore, dumbBlobStore, new BucketNameGenerator(clock), clock,
RetentionConfiguration.DEFAULT));
InMemoryIntegrationResources inMemoryResource = InMemoryIntegrationResources.defaultResources();
mailboxManager = spy(inMemoryResource.getMailboxManager());
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org