You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2021/09/06 02:52:46 UTC
[james-project] branch master updated: JAMES-3150 Integrate Blob
garbage collection to WebAdmin (#624)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new 00a4a03 JAMES-3150 Integrate Blob garbage collection to WebAdmin (#624)
00a4a03 is described below
commit 00a4a0334334dc7e1f1d43a2fcb443cb0deb6705
Author: vttran <vt...@linagora.com>
AuthorDate: Mon Sep 6 09:52:42 2021 +0700
JAMES-3150 Integrate Blob garbage collection to WebAdmin (#624)
---
.../james/CassandraRabbitMQJamesServerMain.java | 2 -
server/blob/blob-storage-strategy/pom.xml | 11 +
.../server/blob/deduplication/BlobGCTask.java | 24 +-
.../BlobGCTaskAdditionalInformationDTO.java | 3 +-
.../server/blob/deduplication/BlobGCTaskDTO.java | 94 +++++++
.../BlobGCTaskAdditionalInformationDTOTest.java | 45 ++++
.../deduplication/BlobGCTaskSerializationTest.java | 77 ++++++
.../json/blobGC.additionalInformation.json | 10 +
.../src/test/resources/json/blobGC.task.json | 6 +
.../{distributed => blob/deduplication-gc}/pom.xml | 58 +----
.../blobstore/BlobDeduplicationGCModule.java | 94 +++++++
.../modules/mailbox/CassandraMailboxModule.java | 8 +
server/container/guice/distributed/pom.xml | 4 +
.../modules/blobstore/BlobStoreModulesChooser.java | 11 +-
.../CassandraMailRepositoryModule.java | 5 +
server/container/guice/pom.xml | 6 +
.../blobstore/server/BlobRoutesModules.java | 35 +++
.../modules/queue/rabbitmq/RabbitMQModule.java | 5 +
...abbitMQWebAdminServerBlobGCIntegrationTest.java | 281 +++++++++++++++++++++
...dminServerTaskSerializationIntegrationTest.java | 27 +-
.../test/resources/eml/emailWithOnlyAttachment.eml | 16 ++
21 files changed, 762 insertions(+), 60 deletions(-)
diff --git a/server/apps/distributed-app/src/main/java/org/apache/james/CassandraRabbitMQJamesServerMain.java b/server/apps/distributed-app/src/main/java/org/apache/james/CassandraRabbitMQJamesServerMain.java
index f78c019..58f2d41 100644
--- a/server/apps/distributed-app/src/main/java/org/apache/james/CassandraRabbitMQJamesServerMain.java
+++ b/server/apps/distributed-app/src/main/java/org/apache/james/CassandraRabbitMQJamesServerMain.java
@@ -43,7 +43,6 @@ import org.apache.james.modules.data.CassandraUsersRepositoryModule;
import org.apache.james.modules.event.JMAPEventBusModule;
import org.apache.james.modules.event.RabbitMQEventBusModule;
import org.apache.james.modules.eventstore.CassandraEventStoreModule;
-import org.apache.james.modules.mailbox.BlobStoreAPIModule;
import org.apache.james.modules.mailbox.CassandraDeletedMessageVaultModule;
import org.apache.james.modules.mailbox.CassandraMailboxModule;
import org.apache.james.modules.mailbox.CassandraQuotaMailingModule;
@@ -127,7 +126,6 @@ public class CassandraRabbitMQJamesServerMain implements JamesServerMain {
new CassandraQuotaMailingModule());
private static final Module BLOB_MODULE = Modules.combine(
- new BlobStoreAPIModule(),
new BlobExportMechanismModule());
private static final Module CASSANDRA_EVENT_STORE_JSON_SERIALIZATION_DEFAULT_MODULE = binder ->
diff --git a/server/blob/blob-storage-strategy/pom.xml b/server/blob/blob-storage-strategy/pom.xml
index ceb5a5b..ce924b9 100644
--- a/server/blob/blob-storage-strategy/pom.xml
+++ b/server/blob/blob-storage-strategy/pom.xml
@@ -46,6 +46,12 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>james-json</artifactId>
+ <type>test-jar</type>
+ <scope>json</scope>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>james-server-task-api</artifactId>
</dependency>
<dependency>
@@ -67,6 +73,11 @@
<artifactId>reactor-scala-extensions_${scala.base}</artifactId>
</dependency>
<dependency>
+ <groupId>net.javacrumbs.json-unit</groupId>
+ <artifactId>json-unit-assertj</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-configuration2</artifactId>
</dependency>
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java
index 2177c62..fffdeee 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java
@@ -50,7 +50,8 @@ public class BlobGCTask implements Task {
snapshot.getGcedBlobCount(),
snapshot.getErrorCount(),
snapshot.getBloomFilterExpectedBlobCount(),
- snapshot.getBloomFilterAssociatedProbability());
+ snapshot.getBloomFilterAssociatedProbability(),
+ Clock.systemUTC().instant());
}
private final Instant timestamp;
@@ -66,14 +67,15 @@ public class BlobGCTask implements Task {
long gcedBlobCount,
long errorCount,
long bloomFilterExpectedBlobCount,
- double bloomFilterAssociatedProbability) {
+ double bloomFilterAssociatedProbability,
+ Instant timestamp) {
this.referenceSourceCount = referenceSourceCount;
this.blobCount = blobCount;
this.gcedBlobCount = gcedBlobCount;
this.errorCount = errorCount;
this.bloomFilterExpectedBlobCount = bloomFilterExpectedBlobCount;
this.bloomFilterAssociatedProbability = bloomFilterAssociatedProbability;
- this.timestamp = Clock.systemUTC().instant();
+ this.timestamp = timestamp;
}
@Override
@@ -222,4 +224,20 @@ public class BlobGCTask implements Task {
public Optional<TaskExecutionDetails.AdditionalInformation> details() {
return Optional.of(AdditionalInformation.from(context));
}
+
+ public Clock getClock() {
+ return clock;
+ }
+
+ public BucketName getBucketName() {
+ return bucketName;
+ }
+
+ public int getExpectedBlobCount() {
+ return expectedBlobCount;
+ }
+
+ public double getAssociatedProbability() {
+ return associatedProbability;
+ }
}
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java
index f9da97a..8caf790 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java
@@ -39,7 +39,8 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
dto.gcedBlobCount,
dto.errorCount,
dto.bloomFilterExpectedBlobCount,
- dto.bloomFilterAssociatedProbability
+ dto.bloomFilterAssociatedProbability,
+ dto.timestamp
))
.toDTOConverter((domain, type) ->
new BlobGCTaskAdditionalInformationDTO(
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java
new file mode 100644
index 0000000..cdec43f
--- /dev/null
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java
@@ -0,0 +1,94 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.server.blob.deduplication;
+
+import java.time.Clock;
+import java.util.Set;
+
+import org.apache.james.blob.api.BlobReferenceSource;
+import org.apache.james.blob.api.BlobStoreDAO;
+import org.apache.james.blob.api.BucketName;
+import org.apache.james.json.DTOModule;
+import org.apache.james.server.task.json.dto.TaskDTO;
+import org.apache.james.server.task.json.dto.TaskDTOModule;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+public class BlobGCTaskDTO implements TaskDTO {
+
+ private final String bucketName;
+ private final int expectedBlobCount;
+ private final double associatedProbability;
+ private final String type;
+
+ public BlobGCTaskDTO(@JsonProperty("bucketName") String bucketName,
+ @JsonProperty("expectedBlobCount") int expectedBlobCount,
+ @JsonProperty("associatedProbability") double associatedProbability,
+ @JsonProperty("type") String type) {
+ this.bucketName = bucketName;
+ this.expectedBlobCount = expectedBlobCount;
+ this.associatedProbability = associatedProbability;
+ this.type = type;
+ }
+
+ public static TaskDTOModule<BlobGCTask, BlobGCTaskDTO> module(BlobStoreDAO blobStoreDAO,
+ GenerationAwareBlobId.Factory generationAwareBlobIdFactory,
+ GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration,
+ Set<BlobReferenceSource> blobReferenceSources,
+ Clock clock) {
+ return DTOModule.forDomainObject(BlobGCTask.class)
+ .convertToDTO(BlobGCTaskDTO.class)
+ .toDomainObjectConverter(dto ->
+ BlobGCTask.builder()
+ .blobStoreDAO(blobStoreDAO)
+ .generationAwareBlobIdFactory(generationAwareBlobIdFactory)
+ .generationAwareBlobIdConfiguration(generationAwareBlobIdConfiguration)
+ .blobReferenceSource(blobReferenceSources)
+ .bucketName(BucketName.of(dto.bucketName))
+ .clock(clock)
+ .expectedBlobCount(dto.expectedBlobCount)
+ .associatedProbability(dto.associatedProbability))
+ .toDTOConverter((domain, type) ->
+ new BlobGCTaskDTO(
+ domain.getBucketName().asString(),
+ domain.getExpectedBlobCount(),
+ domain.getAssociatedProbability(),
+ type))
+ .typeName(BlobGCTask.TASK_TYPE.asString())
+ .withFactory(TaskDTOModule::new);
+ }
+
+ @Override
+ public String getType() {
+ return type;
+ }
+
+ public String getBucketName() {
+ return bucketName;
+ }
+
+ public int getExpectedBlobCount() {
+ return expectedBlobCount;
+ }
+
+ public double getAssociatedProbability() {
+ return associatedProbability;
+ }
+}
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java
new file mode 100644
index 0000000..c2b96d1
--- /dev/null
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java
@@ -0,0 +1,45 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.server.blob.deduplication;
+
+import java.time.Instant;
+
+import org.apache.james.JsonSerializationVerifier;
+import org.apache.james.util.ClassLoaderUtils;
+import org.junit.jupiter.api.Test;
+
+public class BlobGCTaskAdditionalInformationDTOTest {
+
+ @Test
+ void shouldMatchJsonSerializationContract() throws Exception {
+ JsonSerializationVerifier.dtoModule(BlobGCTaskAdditionalInformationDTO.SERIALIZATION_MODULE)
+ .bean(new BlobGCTask.AdditionalInformation(
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 0.8,
+ Instant.parse("2007-12-03T10:15:30.00Z")
+ ))
+ .json(ClassLoaderUtils.getSystemResourceAsString("json/blobGC.additionalInformation.json"))
+ .verify();
+ }
+}
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java
new file mode 100644
index 0000000..0cb40d7
--- /dev/null
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java
@@ -0,0 +1,77 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.server.blob.deduplication;
+
+import static org.mockito.Mockito.mock;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Set;
+
+import org.apache.james.JsonSerializationVerifier;
+import org.apache.james.blob.api.BlobReferenceSource;
+import org.apache.james.blob.api.BlobStoreDAO;
+import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.util.ClassLoaderUtils;
+import org.apache.james.utils.UpdatableTickingClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+public class BlobGCTaskSerializationTest {
+ BlobStoreDAO blobStoreDAO;
+ GenerationAwareBlobId.Factory generationAwareBlobIdFactory;
+ GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration;
+ Set<BlobReferenceSource> blobReferenceSources;
+ Clock clock;
+
+ @BeforeEach
+ void setUp() {
+ blobStoreDAO = mock(BlobStoreDAO.class);
+ blobReferenceSources = ImmutableSet.of(mock(BlobReferenceSource.class));
+ clock = new UpdatableTickingClock(Instant.parse("2007-12-03T10:15:30.00Z"));
+ generationAwareBlobIdConfiguration = GenerationAwareBlobId.Configuration.DEFAULT;
+ generationAwareBlobIdFactory = new GenerationAwareBlobId.Factory(clock, new HashBlobId.Factory(), generationAwareBlobIdConfiguration);
+ }
+
+ @Test
+ void shouldMatchJsonSerializationContract() throws Exception {
+ JsonSerializationVerifier.dtoModule(BlobGCTaskDTO.module(
+ blobStoreDAO,
+ generationAwareBlobIdFactory,
+ generationAwareBlobIdConfiguration,
+ blobReferenceSources,
+ clock))
+ .bean(new BlobGCTask(
+ blobStoreDAO,
+ generationAwareBlobIdFactory,
+ generationAwareBlobIdConfiguration,
+ blobReferenceSources,
+ BucketName.DEFAULT,
+ clock,
+ 99,
+ 0.8
+ ))
+ .json(ClassLoaderUtils.getSystemResourceAsString("json/blobGC.task.json"))
+ .verify();
+ }
+}
diff --git a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json
new file mode 100644
index 0000000..ba4ddee
--- /dev/null
+++ b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json
@@ -0,0 +1,10 @@
+{
+ "type": "BlobGCTask",
+ "timestamp": "2007-12-03T10:15:30Z",
+ "referenceSourceCount": 1,
+ "blobCount": 2,
+ "gcedBlobCount": 3,
+ "errorCount": 4,
+ "bloomFilterExpectedBlobCount": 5,
+ "bloomFilterAssociatedProbability": 0.8
+}
\ No newline at end of file
diff --git a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json
new file mode 100644
index 0000000..44f1dac
--- /dev/null
+++ b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json
@@ -0,0 +1,6 @@
+{
+ "associatedProbability": 0.8,
+ "bucketName": "default",
+ "expectedBlobCount": 99,
+ "type": "BlobGCTask"
+}
\ No newline at end of file
diff --git a/server/container/guice/distributed/pom.xml b/server/container/guice/blob/deduplication-gc/pom.xml
similarity index 50%
copy from server/container/guice/distributed/pom.xml
copy to server/container/guice/blob/deduplication-gc/pom.xml
index 51e3b1d..f776fd0 100644
--- a/server/container/guice/distributed/pom.xml
+++ b/server/container/guice/blob/deduplication-gc/pom.xml
@@ -25,79 +25,43 @@
<groupId>org.apache.james</groupId>
<artifactId>james-server-guice</artifactId>
<version>3.7.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>james-server-guice-distributed</artifactId>
+ <artifactId>blob-deduplication-gc-guice</artifactId>
<packaging>jar</packaging>
- <name>Apache James :: Server :: Distributed - guice modules</name>
- <description>Guice injections for the Distributed messaging technologies</description>
-
- <properties>
- <cassandra.includes>empty</cassandra.includes>
- </properties>
+ <name>Apache James :: Server :: Blob Storage Strategy - guice injection</name>
<dependencies>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>apache-james-mailbox-event-json</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>apache-james-mailbox-tools-quota-recompute</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>blob-aes</artifactId>
+ <artifactId>apache-james-mailbox-cassandra</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>blob-s3-guice</artifactId>
+ <artifactId>blob-memory</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>event-bus-distributed</artifactId>
+ <artifactId>blob-storage-strategy</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>event-sourcing-event-store-memory</artifactId>
- <scope>test</scope>
+ <artifactId>james-server-guice-webadmin-data</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>james-server-guice-cassandra</artifactId>
+ <artifactId>james-server-webadmin-core</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>james-server-guice-common</artifactId>
- <type>test-jar</type>
+ <artifactId>testing-base</artifactId>
<scope>test</scope>
</dependency>
<dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>james-server-jmap</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>james-server-jmap-rfc-8621</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>james-server-task-distributed</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>james-server-webadmin-rabbitmq</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>queue-rabbitmq-guice</artifactId>
- </dependency>
- <dependency>
- <groupId>${james.groupId}</groupId>
- <artifactId>testing-base</artifactId>
- <scope>test</scope>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
</dependency>
</dependencies>
</project>
diff --git a/server/container/guice/blob/deduplication-gc/src/main/java/org/apache/james/modules/blobstore/BlobDeduplicationGCModule.java b/server/container/guice/blob/deduplication-gc/src/main/java/org/apache/james/modules/blobstore/BlobDeduplicationGCModule.java
new file mode 100644
index 0000000..41eab46
--- /dev/null
+++ b/server/container/guice/blob/deduplication-gc/src/main/java/org/apache/james/modules/blobstore/BlobDeduplicationGCModule.java
@@ -0,0 +1,94 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules.blobstore;
+
+import java.time.Clock;
+import java.util.Set;
+
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BlobReferenceSource;
+import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.BlobStoreDAO;
+import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.blob.api.MetricableBlobStore;
+import org.apache.james.modules.blobstore.server.BlobRoutesModules;
+import org.apache.james.server.blob.deduplication.BlobGCTaskAdditionalInformationDTO;
+import org.apache.james.server.blob.deduplication.BlobGCTaskDTO;
+import org.apache.james.server.blob.deduplication.GenerationAwareBlobId;
+import org.apache.james.server.task.json.dto.AdditionalInformationDTO;
+import org.apache.james.server.task.json.dto.AdditionalInformationDTOModule;
+import org.apache.james.server.task.json.dto.TaskDTO;
+import org.apache.james.server.task.json.dto.TaskDTOModule;
+import org.apache.james.task.Task;
+import org.apache.james.task.TaskExecutionDetails;
+import org.apache.james.webadmin.dto.DTOModuleInjections;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.Provides;
+import com.google.inject.Scopes;
+import com.google.inject.Singleton;
+import com.google.inject.multibindings.ProvidesIntoSet;
+import com.google.inject.name.Named;
+
+public class BlobDeduplicationGCModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ bind(HashBlobId.Factory.class).in(Scopes.SINGLETON);
+ bind(BlobId.Factory.class).to(GenerationAwareBlobId.Factory.class);
+
+ bind(MetricableBlobStore.class).in(Scopes.SINGLETON);
+ bind(BlobStore.class).to(MetricableBlobStore.class);
+
+ install(new BlobRoutesModules());
+ }
+
+ @Singleton
+ @Provides
+ public GenerationAwareBlobId.Factory generationAwareBlobIdFactory(Clock clock, HashBlobId.Factory delegate, GenerationAwareBlobId.Configuration configuration) {
+ return new GenerationAwareBlobId.Factory(clock, delegate, configuration);
+ }
+
+ @Singleton
+ @Provides
+ public GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration() {
+ return GenerationAwareBlobId.Configuration.DEFAULT;
+ }
+
+ @ProvidesIntoSet
+ public TaskDTOModule<? extends Task, ? extends TaskDTO> blobGCTask(BlobStoreDAO blobStoreDAO,
+ GenerationAwareBlobId.Factory generationAwareBlobIdFactory,
+ GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration,
+ Set<BlobReferenceSource> blobReferenceSources,
+ Clock clock) {
+ return BlobGCTaskDTO.module(blobStoreDAO, generationAwareBlobIdFactory, generationAwareBlobIdConfiguration, blobReferenceSources, clock);
+ }
+
+ @ProvidesIntoSet
+ public AdditionalInformationDTOModule<? extends TaskExecutionDetails.AdditionalInformation, ? extends AdditionalInformationDTO> blobGCAdditionalInformation() {
+ return BlobGCTaskAdditionalInformationDTO.SERIALIZATION_MODULE;
+ }
+
+ @Named(DTOModuleInjections.WEBADMIN_DTO)
+ @ProvidesIntoSet
+ public AdditionalInformationDTOModule<? extends TaskExecutionDetails.AdditionalInformation, ? extends AdditionalInformationDTO> webAdminBlobGCAdditionalInformation() {
+ return BlobGCTaskAdditionalInformationDTO.SERIALIZATION_MODULE;
+ }
+}
diff --git a/server/container/guice/cassandra/src/main/java/org/apache/james/modules/mailbox/CassandraMailboxModule.java b/server/container/guice/cassandra/src/main/java/org/apache/james/modules/mailbox/CassandraMailboxModule.java
index 867d7db..908741f 100644
--- a/server/container/guice/cassandra/src/main/java/org/apache/james/modules/mailbox/CassandraMailboxModule.java
+++ b/server/container/guice/cassandra/src/main/java/org/apache/james/modules/mailbox/CassandraMailboxModule.java
@@ -25,6 +25,7 @@ import javax.inject.Singleton;
import org.apache.james.adapter.mailbox.UserRepositoryAuthenticator;
import org.apache.james.adapter.mailbox.UserRepositoryAuthorizator;
import org.apache.james.backends.cassandra.components.CassandraModule;
+import org.apache.james.blob.api.BlobReferenceSource;
import org.apache.james.events.EventListener;
import org.apache.james.eventsourcing.Event;
import org.apache.james.eventsourcing.eventstore.cassandra.dto.EventDTO;
@@ -55,6 +56,7 @@ import org.apache.james.mailbox.cassandra.DeleteMessageListener;
import org.apache.james.mailbox.cassandra.ids.CassandraId;
import org.apache.james.mailbox.cassandra.ids.CassandraMessageId;
import org.apache.james.mailbox.cassandra.mail.ACLMapper;
+import org.apache.james.mailbox.cassandra.mail.AttachmentBlobReferenceSource;
import org.apache.james.mailbox.cassandra.mail.CassandraACLDAOV1;
import org.apache.james.mailbox.cassandra.mail.CassandraACLDAOV2;
import org.apache.james.mailbox.cassandra.mail.CassandraACLMapper;
@@ -75,6 +77,7 @@ import org.apache.james.mailbox.cassandra.mail.CassandraMessageIdToImapUidDAO;
import org.apache.james.mailbox.cassandra.mail.CassandraModSeqProvider;
import org.apache.james.mailbox.cassandra.mail.CassandraUidProvider;
import org.apache.james.mailbox.cassandra.mail.CassandraUserMailboxRightsDAO;
+import org.apache.james.mailbox.cassandra.mail.MessageBlobReferenceSource;
import org.apache.james.mailbox.cassandra.mail.eventsourcing.acl.ACLModule;
import org.apache.james.mailbox.cassandra.modules.CassandraAclModule;
import org.apache.james.mailbox.cassandra.modules.CassandraAnnotationModule;
@@ -230,6 +233,11 @@ public class CassandraMailboxModule extends AbstractModule {
Multibinder.newSetBinder(binder(), new TypeLiteral<EventDTOModule<? extends Event, ? extends EventDTO>>() {})
.addBinding().toInstance(ACLModule.ACL_UPDATE);
+
+ Multibinder.newSetBinder(binder(), BlobReferenceSource.class)
+ .addBinding().to(AttachmentBlobReferenceSource.class);
+ Multibinder.newSetBinder(binder(), BlobReferenceSource.class)
+ .addBinding().to(MessageBlobReferenceSource.class);
}
@Singleton
diff --git a/server/container/guice/distributed/pom.xml b/server/container/guice/distributed/pom.xml
index 51e3b1d..9675ab6 100644
--- a/server/container/guice/distributed/pom.xml
+++ b/server/container/guice/distributed/pom.xml
@@ -53,6 +53,10 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-deduplication-gc-guice</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>blob-s3-guice</artifactId>
</dependency>
<dependency>
diff --git a/server/container/guice/distributed/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java b/server/container/guice/distributed/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
index 7e4680f..34fa484 100644
--- a/server/container/guice/distributed/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
+++ b/server/container/guice/distributed/src/main/java/org/apache/james/modules/blobstore/BlobStoreModulesChooser.java
@@ -35,6 +35,7 @@ import org.apache.james.eventsourcing.eventstore.cassandra.dto.EventDTOModule;
import org.apache.james.lifecycle.api.StartUpCheck;
import org.apache.james.modules.blobstore.validation.EventsourcingStorageStrategy;
import org.apache.james.modules.blobstore.validation.StorageStrategyModule;
+import org.apache.james.modules.mailbox.BlobStoreAPIModule;
import org.apache.james.modules.mailbox.CassandraBlobStoreDependenciesModule;
import org.apache.james.modules.mailbox.CassandraBucketModule;
import org.apache.james.modules.objectstorage.DefaultBucketModule;
@@ -128,7 +129,7 @@ public class BlobStoreModulesChooser {
return ImmutableList.<Module>builder()
.add(chooseEncryptionModule(choosingConfiguration.getCryptoConfig()))
.add(chooseBlobStoreDAOModule(choosingConfiguration.getImplementation()))
- .add(chooseStoragePolicyModule(choosingConfiguration.storageStrategy()))
+ .addAll(chooseStoragePolicyModule(choosingConfiguration.storageStrategy()))
.add(new StoragePolicyConfigurationSanityEnforcementModule(choosingConfiguration))
.build();
}
@@ -149,16 +150,18 @@ public class BlobStoreModulesChooser {
return encryptionModule.orElse(new NoEncryptionModule());
}
- private static Module chooseStoragePolicyModule(StorageStrategy storageStrategy) {
+ private static List<Module> chooseStoragePolicyModule(StorageStrategy storageStrategy) {
switch (storageStrategy) {
case DEDUPLICATION:
- return binder -> binder.bind(BlobStore.class)
+ Module deduplicationBlobModule = binder -> binder.bind(BlobStore.class)
.annotatedWith(Names.named(CachedBlobStore.BACKEND))
.to(DeDuplicationBlobStore.class);
+ return ImmutableList.of(new BlobDeduplicationGCModule(), deduplicationBlobModule);
case PASSTHROUGH:
- return binder -> binder.bind(BlobStore.class)
+ Module passThroughBlobModule = binder -> binder.bind(BlobStore.class)
.annotatedWith(Names.named(CachedBlobStore.BACKEND))
.to(PassThroughBlobStore.class);
+ return ImmutableList.of(new BlobStoreAPIModule(), passThroughBlobModule);
default:
throw new RuntimeException("Unknown storage strategy " + storageStrategy.name());
}
diff --git a/server/container/guice/mailrepository-cassandra/src/main/java/org/apache/james/modules/mailrepository/CassandraMailRepositoryModule.java b/server/container/guice/mailrepository-cassandra/src/main/java/org/apache/james/modules/mailrepository/CassandraMailRepositoryModule.java
index 611ed28..242424e 100644
--- a/server/container/guice/mailrepository-cassandra/src/main/java/org/apache/james/modules/mailrepository/CassandraMailRepositoryModule.java
+++ b/server/container/guice/mailrepository-cassandra/src/main/java/org/apache/james/modules/mailrepository/CassandraMailRepositoryModule.java
@@ -21,6 +21,7 @@ package org.apache.james.modules.mailrepository;
import org.apache.commons.configuration2.BaseHierarchicalConfiguration;
import org.apache.james.backends.cassandra.components.CassandraModule;
+import org.apache.james.blob.api.BlobReferenceSource;
import org.apache.james.mailrepository.api.MailRepositoryUrlStore;
import org.apache.james.mailrepository.api.Protocol;
import org.apache.james.mailrepository.cassandra.CassandraMailRepository;
@@ -29,6 +30,7 @@ import org.apache.james.mailrepository.cassandra.CassandraMailRepositoryKeysDAO;
import org.apache.james.mailrepository.cassandra.CassandraMailRepositoryMailDaoV2;
import org.apache.james.mailrepository.cassandra.CassandraMailRepositoryUrlModule;
import org.apache.james.mailrepository.cassandra.CassandraMailRepositoryUrlStore;
+import org.apache.james.mailrepository.cassandra.MailRepositoryBlobReferenceSource;
import org.apache.james.mailrepository.memory.MailRepositoryStoreConfiguration;
import com.google.common.collect.ImmutableList;
@@ -57,6 +59,9 @@ public class CassandraMailRepositoryModule extends AbstractModule {
Multibinder<CassandraModule> cassandraModuleBinder = Multibinder.newSetBinder(binder(), CassandraModule.class);
cassandraModuleBinder.addBinding().toInstance(org.apache.james.mailrepository.cassandra.CassandraMailRepositoryModule.MODULE);
cassandraModuleBinder.addBinding().toInstance(CassandraMailRepositoryUrlModule.MODULE);
+
+ Multibinder.newSetBinder(binder(), BlobReferenceSource.class)
+ .addBinding().to(MailRepositoryBlobReferenceSource.class);
}
}
diff --git a/server/container/guice/pom.xml b/server/container/guice/pom.xml
index dea05b8..125686c 100644
--- a/server/container/guice/pom.xml
+++ b/server/container/guice/pom.xml
@@ -34,6 +34,7 @@
<modules>
<module>blob/api</module>
+ <module>blob/deduplication-gc</module>
<module>blob/export</module>
<module>blob/memory</module>
<module>blob/s3</module>
@@ -84,6 +85,11 @@
<dependencies>
<dependency>
<groupId>${james.groupId}</groupId>
+ <artifactId>blob-deduplication-gc-guice</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
<artifactId>data-cassandra</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/server/container/guice/protocols/webadmin-data/src/main/java/org/apache/james/modules/blobstore/server/BlobRoutesModules.java b/server/container/guice/protocols/webadmin-data/src/main/java/org/apache/james/modules/blobstore/server/BlobRoutesModules.java
new file mode 100644
index 0000000..cc6cf63
--- /dev/null
+++ b/server/container/guice/protocols/webadmin-data/src/main/java/org/apache/james/modules/blobstore/server/BlobRoutesModules.java
@@ -0,0 +1,35 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.modules.blobstore.server;
+
+import org.apache.james.webadmin.Routes;
+import org.apache.james.webadmin.routes.BlobRoutes;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.multibindings.Multibinder;
+
+public class BlobRoutesModules extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ Multibinder<Routes> routesMultiBinder = Multibinder.newSetBinder(binder(), Routes.class);
+ routesMultiBinder.addBinding().to(BlobRoutes.class);
+ }
+}
diff --git a/server/container/guice/queue/rabbitmq/src/main/java/org/apache/james/modules/queue/rabbitmq/RabbitMQModule.java b/server/container/guice/queue/rabbitmq/src/main/java/org/apache/james/modules/queue/rabbitmq/RabbitMQModule.java
index d7adf09..e8b2866 100644
--- a/server/container/guice/queue/rabbitmq/src/main/java/org/apache/james/modules/queue/rabbitmq/RabbitMQModule.java
+++ b/server/container/guice/queue/rabbitmq/src/main/java/org/apache/james/modules/queue/rabbitmq/RabbitMQModule.java
@@ -32,6 +32,7 @@ import org.apache.james.backends.rabbitmq.RabbitMQHealthCheck;
import org.apache.james.backends.rabbitmq.ReactorRabbitMQChannelPool;
import org.apache.james.backends.rabbitmq.ReceiverProvider;
import org.apache.james.backends.rabbitmq.SimpleConnectionPool;
+import org.apache.james.blob.api.BlobReferenceSource;
import org.apache.james.core.healthcheck.HealthCheck;
import org.apache.james.eventsourcing.Event;
import org.apache.james.eventsourcing.eventstore.cassandra.dto.EventDTO;
@@ -54,6 +55,7 @@ import org.apache.james.queue.rabbitmq.view.cassandra.CassandraMailQueueViewStar
import org.apache.james.queue.rabbitmq.view.cassandra.ContentStartDAO;
import org.apache.james.queue.rabbitmq.view.cassandra.DeletedMailsDAO;
import org.apache.james.queue.rabbitmq.view.cassandra.EnqueuedMailsDAO;
+import org.apache.james.queue.rabbitmq.view.cassandra.MailQueueViewBlobReferenceSource;
import org.apache.james.queue.rabbitmq.view.cassandra.configuration.CassandraMailQueueViewConfiguration;
import org.apache.james.queue.rabbitmq.view.cassandra.configuration.CassandraMailQueueViewConfigurationModule;
import org.apache.james.queue.rabbitmq.view.cassandra.configuration.EventsourcingConfigurationManagement;
@@ -103,6 +105,9 @@ public class RabbitMQModule extends AbstractModule {
Multibinder<SimpleConnectionPool.ReconnectionHandler> reconnectionHandlerMultibinder = Multibinder.newSetBinder(binder(), SimpleConnectionPool.ReconnectionHandler.class);
reconnectionHandlerMultibinder.addBinding().to(SpoolerReconnectionHandler.class);
+
+ Multibinder.newSetBinder(binder(), BlobReferenceSource.class)
+ .addBinding().to(MailQueueViewBlobReferenceSource.class);
}
@Provides
diff --git a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerBlobGCIntegrationTest.java b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerBlobGCIntegrationTest.java
new file mode 100644
index 0000000..320f32d
--- /dev/null
+++ b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerBlobGCIntegrationTest.java
@@ -0,0 +1,281 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.webadmin.integration.rabbitmq;
+
+import static io.restassured.RestAssured.given;
+import static io.restassured.RestAssured.with;
+import static org.hamcrest.Matchers.is;
+
+import java.time.Clock;
+import java.time.ZonedDateTime;
+import java.util.Date;
+
+import javax.mail.Flags;
+import javax.mail.util.SharedByteArrayInputStream;
+
+import org.apache.james.CassandraExtension;
+import org.apache.james.CassandraRabbitMQJamesConfiguration;
+import org.apache.james.CassandraRabbitMQJamesServerMain;
+import org.apache.james.DockerElasticSearchExtension;
+import org.apache.james.GuiceJamesServer;
+import org.apache.james.GuiceModuleTestExtension;
+import org.apache.james.JamesServerBuilder;
+import org.apache.james.JamesServerExtension;
+import org.apache.james.SearchConfiguration;
+import org.apache.james.core.Username;
+import org.apache.james.mailbox.exception.MailboxException;
+import org.apache.james.mailbox.model.MailboxConstants;
+import org.apache.james.mailbox.model.MailboxPath;
+import org.apache.james.mailbox.probe.MailboxProbe;
+import org.apache.james.modules.AwsS3BlobStoreExtension;
+import org.apache.james.modules.MailboxProbeImpl;
+import org.apache.james.modules.RabbitMQExtension;
+import org.apache.james.modules.blobstore.BlobStoreConfiguration;
+import org.apache.james.probe.DataProbe;
+import org.apache.james.task.TaskManager;
+import org.apache.james.util.ClassLoaderUtils;
+import org.apache.james.utils.DataProbeImpl;
+import org.apache.james.utils.UpdatableTickingClock;
+import org.apache.james.utils.WebAdminGuiceProbe;
+import org.apache.james.webadmin.WebAdminUtils;
+import org.apache.james.webadmin.routes.TasksRoutes;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import com.google.inject.Module;
+
+import io.restassured.RestAssured;
+
+public class RabbitMQWebAdminServerBlobGCIntegrationTest {
+ private static final ZonedDateTime TIMESTAMP = ZonedDateTime.parse("2015-10-30T16:12:00Z");
+
+ public static class ClockExtension implements GuiceModuleTestExtension {
+ private UpdatableTickingClock clock;
+
+ @Override
+ public void beforeEach(ExtensionContext extensionContext) {
+ clock = new UpdatableTickingClock(TIMESTAMP.toInstant());
+ }
+
+ @Override
+ public Module getModule() {
+ return binder -> binder.bind(Clock.class).toInstance(clock);
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return parameterContext.getParameter().getType() == UpdatableTickingClock.class;
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return clock;
+ }
+ }
+
+ @RegisterExtension
+ static JamesServerExtension testExtension = new JamesServerBuilder<CassandraRabbitMQJamesConfiguration>(tmpDir ->
+ CassandraRabbitMQJamesConfiguration.builder()
+ .workingDirectory(tmpDir)
+ .configurationFromClasspath()
+ .blobStore(BlobStoreConfiguration.builder()
+ .s3()
+ .disableCache()
+ .deduplication()
+ .noCryptoConfig())
+ .searchConfiguration(SearchConfiguration.elasticSearch())
+ .build())
+ .extension(new DockerElasticSearchExtension())
+ .extension(new CassandraExtension())
+ .extension(new AwsS3BlobStoreExtension())
+ .extension(new RabbitMQExtension())
+ .extension(new ClockExtension())
+ .server(CassandraRabbitMQJamesServerMain::createServer)
+ .build();
+
+ private static final String DOMAIN = "domain";
+ private static final String USERNAME = "username@" + DOMAIN;
+
+ private DataProbe dataProbe;
+ private MailboxProbe mailboxProbe;
+
+ @BeforeEach
+ void setUp(GuiceJamesServer guiceJamesServer, UpdatableTickingClock clock) throws Exception {
+ clock.setInstant(TIMESTAMP.toInstant());
+
+ WebAdminGuiceProbe webAdminGuiceProbe = guiceJamesServer.getProbe(WebAdminGuiceProbe.class);
+ dataProbe = guiceJamesServer.getProbe(DataProbeImpl.class);
+ mailboxProbe = guiceJamesServer.getProbe(MailboxProbeImpl.class);
+
+ dataProbe.addDomain(DOMAIN);
+ dataProbe.addUser(USERNAME, "secret");
+ mailboxProbe.createMailbox(MailboxConstants.USER_NAMESPACE, USERNAME, MailboxConstants.INBOX);
+
+ RestAssured.requestSpecification = WebAdminUtils.buildRequestSpecification(webAdminGuiceProbe.getWebAdminPort())
+ .build();
+ RestAssured.enableLoggingOfRequestAndResponseIfValidationFails();
+ }
+
+ @Test
+ void blobGCShouldRemoveUnreferencedAndInactiveBlobId(UpdatableTickingClock clock) throws MailboxException {
+ SharedByteArrayInputStream mailInputStream = ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithOnlyAttachment.eml");
+ mailboxProbe.appendMessage(
+ USERNAME,
+ MailboxPath.inbox(Username.of(USERNAME)),
+ mailInputStream,
+ new Date(),
+ false,
+ new Flags());
+
+ mailboxProbe.deleteMailbox(MailboxConstants.USER_NAMESPACE, USERNAME, MailboxConstants.INBOX);
+ clock.setInstant(TIMESTAMP.plusMonths(2).toInstant());
+
+ String taskId = given()
+ .queryParam("scope", "unreferenced")
+ .delete("blobs")
+ .jsonPath()
+ .getString("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is(TaskManager.Status.COMPLETED.getValue()))
+ .body("taskId", is(taskId))
+ .body("type", is("BlobGCTask"))
+ .body("additionalInformation.referenceSourceCount", is(0))
+ .body("additionalInformation.blobCount", is(3))
+ .body("additionalInformation.gcedBlobCount", is(3))
+ .body("additionalInformation.errorCount", is(0));
+ }
+
+ @Test
+ void blobGCShouldNotRemoveActiveBlobId() throws MailboxException {
+ SharedByteArrayInputStream mailInputStream = ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithOnlyAttachment.eml");
+ mailboxProbe.appendMessage(
+ USERNAME,
+ MailboxPath.inbox(Username.of(USERNAME)),
+ mailInputStream,
+ new Date(),
+ false,
+ new Flags());
+
+ mailboxProbe.deleteMailbox(MailboxConstants.USER_NAMESPACE, USERNAME, MailboxConstants.INBOX);
+
+ String taskId = given()
+ .queryParam("scope", "unreferenced")
+ .delete("blobs")
+ .jsonPath()
+ .getString("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is(TaskManager.Status.COMPLETED.getValue()))
+ .body("taskId", is(taskId))
+ .body("type", is("BlobGCTask"))
+ .body("additionalInformation.referenceSourceCount", is(0))
+ .body("additionalInformation.blobCount", is(3))
+ .body("additionalInformation.gcedBlobCount", is(0))
+ .body("additionalInformation.errorCount", is(0));
+ }
+
+ @Test
+ void blobGCShouldNotRemoveReferencedBlobId(UpdatableTickingClock clock) throws MailboxException {
+ SharedByteArrayInputStream mailInputStream = ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithOnlyAttachment.eml");
+ mailboxProbe.appendMessage(
+ USERNAME,
+ MailboxPath.inbox(Username.of(USERNAME)),
+ mailInputStream,
+ new Date(),
+ false,
+ new Flags());
+ clock.setInstant(TIMESTAMP.plusMonths(2).toInstant());
+
+ String taskId = given()
+ .queryParam("scope", "unreferenced")
+ .delete("blobs")
+ .jsonPath()
+ .getString("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is(TaskManager.Status.COMPLETED.getValue()))
+ .body("taskId", is(taskId))
+ .body("type", is("BlobGCTask"))
+ .body("additionalInformation.referenceSourceCount", is(3))
+ .body("additionalInformation.blobCount", is(3))
+ .body("additionalInformation.gcedBlobCount", is(0))
+ .body("additionalInformation.errorCount", is(0));
+ }
+
+ @Test
+ void blobGCShouldNotRemoveReferencedBlobIdToAnotherMailbox(UpdatableTickingClock clock) throws Exception {
+ SharedByteArrayInputStream mailInputStream = ClassLoaderUtils.getSystemResourceAsSharedStream("eml/emailWithOnlyAttachment.eml");
+ mailboxProbe.appendMessage(
+ USERNAME,
+ MailboxPath.inbox(Username.of(USERNAME)),
+ mailInputStream,
+ new Date(),
+ false,
+ new Flags());
+
+ mailboxProbe.createMailbox(MailboxConstants.USER_NAMESPACE, USERNAME, "CustomBox");
+ mailboxProbe.appendMessage(
+ USERNAME,
+ MailboxPath.forUser(Username.of(USERNAME), "CustomBox"),
+ mailInputStream,
+ new Date(),
+ false,
+ new Flags());
+
+ mailboxProbe.deleteMailbox(MailboxConstants.USER_NAMESPACE, USERNAME, MailboxConstants.INBOX);
+ clock.setInstant(TIMESTAMP.plusMonths(2).toInstant());
+
+ String taskId = given()
+ .queryParam("scope", "unreferenced")
+ .delete("blobs")
+ .jsonPath()
+ .getString("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is(TaskManager.Status.COMPLETED.getValue()))
+ .body("taskId", is(taskId))
+ .body("type", is("BlobGCTask"))
+ .body("additionalInformation.referenceSourceCount", is(2))
+ .body("additionalInformation.blobCount", is(3))
+ .body("additionalInformation.gcedBlobCount", is(2))
+ .body("additionalInformation.errorCount", is(0));
+ }
+}
diff --git a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerTaskSerializationIntegrationTest.java b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerTaskSerializationIntegrationTest.java
index 7a072d2..a6a0133 100644
--- a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerTaskSerializationIntegrationTest.java
+++ b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/RabbitMQWebAdminServerTaskSerializationIntegrationTest.java
@@ -32,7 +32,6 @@ import static org.hamcrest.collection.IsMapWithSize.anEmptyMap;
import java.io.ByteArrayInputStream;
import java.util.Date;
-import java.util.stream.IntStream;
import java.util.stream.Stream;
import javax.mail.Flags;
@@ -89,8 +88,6 @@ import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
-import com.github.fge.lambdas.Throwing;
-
import io.restassured.RestAssured;
import io.restassured.http.ContentType;
@@ -733,6 +730,30 @@ class RabbitMQWebAdminServerTaskSerializationIntegrationTest {
.body("additionalInformation.scope", is("expired"));
}
+ @Test
+ void blobGCTaskShouldComplete() {
+ String taskId = given()
+ .queryParam("scope", "unreferenced")
+ .delete("blobs")
+ .jsonPath()
+ .getString("taskId");
+
+ with()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("status", is(TaskManager.Status.COMPLETED.getValue()))
+ .body("taskId", is(taskId))
+ .body("type", is("BlobGCTask"))
+ .body("additionalInformation.referenceSourceCount", is(0))
+ .body("additionalInformation.blobCount", is(0))
+ .body("additionalInformation.gcedBlobCount", is(0))
+ .body("additionalInformation.errorCount", is(0))
+ .body("additionalInformation.bloomFilterExpectedBlobCount", is(1000000))
+ .body("additionalInformation.bloomFilterAssociatedProbability", is(0.8F));
+ }
+
private MailboxAdded createMailboxAdded() {
String uuid = "6e0dd59d-660e-4d9b-b22f-0354479f47b4";
return EventFactory.mailboxAdded()
diff --git a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/resources/eml/emailWithOnlyAttachment.eml b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/resources/eml/emailWithOnlyAttachment.eml
new file mode 100644
index 0000000..452d4cc
--- /dev/null
+++ b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/resources/eml/emailWithOnlyAttachment.eml
@@ -0,0 +1,16 @@
+Return-Path: <ab...@apache.org>
+Subject: 29989 btellier
+From: <an...@mail.com>
+Content-Disposition: attachment
+MIME-Version: 1.0
+Date: Sun, 02 Apr 2017 22:09:04 -0000
+Content-Type: application/zip; name="9559333830.zip"
+To: <ab...@apache.org>
+Message-ID: <14...@any.com>
+Content-Transfer-Encoding: base64
+
+UEsDBBQAAgAIAEQeg0oN2YT/EAsAAMsWAAAIABwAMjIwODUuanNVVAkAAxBy4VgQcuFYdXgLAAEE
+AAAAAAQAAAAApZhbi1zHFYWfY/B/MP3i7kwj1/2CokAwBPIQ+sGPkgJ1tURkdeiMbYzQf8+3q8+M
+ZmQllgn2aHrqnNq1L2uvtavnj2/b7evz26/Op5M6q/P+8OUX77784g8/lQtLisXTU/68vfzCv/Lg
+D9vqs/3b8fNXf92273ey4XTCykk9w9LpfD7tX+zGzU83b8pPg39uBr/Kmxe7w9PLuP3xwpFKTJ32
+AAEEAAAAAAQAAAAAUEsFBgAAAAABAAEATgAAAFILAAAAAA==
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org