You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2022/11/24 02:01:28 UTC
[james-project] branch master updated: JAMES-3150 Configuration for blob deletion window size (#1328)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
The following commit(s) were added to refs/heads/master by this push:
new fb2fe7da74 JAMES-3150 Configuration for blob deletion window size (#1328)
fb2fe7da74 is described below
commit fb2fe7da74c784240d5b50963a773465277464c5
Author: Benoit TELLIER <bt...@linagora.com>
AuthorDate: Thu Nov 24 09:01:23 2022 +0700
JAMES-3150 Configuration for blob deletion window size (#1328)
---
.../server/blob/deduplication/BlobGCTask.java | 114 ++++++++++++++++-----
.../BlobGCTaskAdditionalInformationDTO.java | 17 ++-
.../server/blob/deduplication/BlobGCTaskDTO.java | 13 ++-
.../blob/deduplication/BloomFilterGCAlgorithm.java | 8 +-
.../BlobGCTaskAdditionalInformationDTOTest.java | 26 ++++-
.../deduplication/BlobGCTaskSerializationTest.java | 29 ++++++
.../BloomFilterGCAlgorithmContract.java | 13 ++-
...on => blobGC-legacy.additionalInformation.json} | 0
.../{blobGC.task.json => blobGC-legacy.task.json} | 0
.../json/blobGC.additionalInformation.json | 3 +-
.../src/test/resources/json/blobGC.task.json | 1 +
.../apache/james/webadmin/routes/BlobRoutes.java | 19 +++-
.../james/webadmin/routes/BlobRoutesTest.java | 18 ++++
13 files changed, 217 insertions(+), 44 deletions(-)
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java
index 807cd88e7f..abfcf70a8b 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTask.java
@@ -37,7 +37,7 @@ public class BlobGCTask implements Task {
public static class AdditionalInformation implements TaskExecutionDetails.AdditionalInformation {
- private static AdditionalInformation from(Context context) {
+ private static AdditionalInformation from(Context context, int deletionWindowSize) {
Context.Snapshot snapshot = context.snapshot();
return new AdditionalInformation(
snapshot.getReferenceSourceCount(),
@@ -46,7 +46,7 @@ public class BlobGCTask implements Task {
snapshot.getErrorCount(),
snapshot.getBloomFilterExpectedBlobCount(),
snapshot.getBloomFilterAssociatedProbability(),
- Clock.systemUTC().instant());
+ Clock.systemUTC().instant(), deletionWindowSize);
}
private final Instant timestamp;
@@ -56,6 +56,7 @@ public class BlobGCTask implements Task {
private final long errorCount;
private final long bloomFilterExpectedBlobCount;
private final double bloomFilterAssociatedProbability;
+ private final int deletionWindowSize;
AdditionalInformation(long referenceSourceCount,
long blobCount,
@@ -63,7 +64,8 @@ public class BlobGCTask implements Task {
long errorCount,
long bloomFilterExpectedBlobCount,
double bloomFilterAssociatedProbability,
- Instant timestamp) {
+ Instant timestamp,
+ int deletionWindowSize) {
this.referenceSourceCount = referenceSourceCount;
this.blobCount = blobCount;
this.gcedBlobCount = gcedBlobCount;
@@ -71,6 +73,7 @@ public class BlobGCTask implements Task {
this.bloomFilterExpectedBlobCount = bloomFilterExpectedBlobCount;
this.bloomFilterAssociatedProbability = bloomFilterAssociatedProbability;
this.timestamp = timestamp;
+ this.deletionWindowSize = deletionWindowSize;
}
@Override
@@ -105,64 +108,117 @@ public class BlobGCTask implements Task {
public double getBloomFilterAssociatedProbability() {
return bloomFilterAssociatedProbability;
}
+
+ public int getDeletionWindowSize() {
+ return deletionWindowSize;
+ }
}
- interface Builder {
+ public static class Builder {
+
+ public static final int DEFAULT_DELETION_WINDOW_SIZE = 1000;
@FunctionalInterface
- interface RequireAssociatedProbability {
- BlobGCTask associatedProbability(double associatedProbability);
+ public interface RequireAssociatedProbability {
+ Builder associatedProbability(double associatedProbability);
}
@FunctionalInterface
- interface RequireExpectedBlobCount {
+ public interface RequireExpectedBlobCount {
RequireAssociatedProbability expectedBlobCount(int expectedBlobCount);
}
@FunctionalInterface
- interface RequireClock {
+ public interface RequireClock {
RequireExpectedBlobCount clock(Clock clock);
}
@FunctionalInterface
- interface RequireBucketName {
+ public interface RequireBucketName {
RequireClock bucketName(BucketName bucketName);
}
@FunctionalInterface
- interface RequireBlobReferenceSources {
+ public interface RequireBlobReferenceSources {
RequireBucketName blobReferenceSource(Set<BlobReferenceSource> blobReferenceSources);
}
@FunctionalInterface
- interface RequireGenerationAwareBlobIdConfiguration {
+ public interface RequireGenerationAwareBlobIdConfiguration {
RequireBlobReferenceSources generationAwareBlobIdConfiguration(GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration);
}
@FunctionalInterface
- interface RequireGenerationAwareBlobIdFactory {
+ public interface RequireGenerationAwareBlobIdFactory {
RequireGenerationAwareBlobIdConfiguration generationAwareBlobIdFactory(GenerationAwareBlobId.Factory generationAwareBlobIdFactory);
}
@FunctionalInterface
- interface RequireBlobStoreDAO {
+ public interface RequireBlobStoreDAO {
RequireGenerationAwareBlobIdFactory blobStoreDAO(BlobStoreDAO blobStoreDAO);
}
+
+ private final BlobStoreDAO blobStoreDAO;
+ private final GenerationAwareBlobId.Factory generationAwareBlobIdFactory;
+ private final GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration;
+ private final Set<BlobReferenceSource> blobReferenceSources;
+ private final Clock clock;
+ private final BucketName bucketName;
+ private final int expectedBlobCount;
+ private final double associatedProbability;
+ private Optional<Integer> deletionWindowSize;
+
+ public Builder(BlobStoreDAO blobStoreDAO, GenerationAwareBlobId.Factory generationAwareBlobIdFactory,
+ GenerationAwareBlobId.Configuration generationAwareBlobIdConfiguration,
+ Set<BlobReferenceSource> blobReferenceSources, Clock clock, BucketName bucketName,
+ int expectedBlobCount, double associatedProbability) {
+ this.blobStoreDAO = blobStoreDAO;
+ this.generationAwareBlobIdFactory = generationAwareBlobIdFactory;
+ this.generationAwareBlobIdConfiguration = generationAwareBlobIdConfiguration;
+ this.blobReferenceSources = blobReferenceSources;
+ this.clock = clock;
+ this.bucketName = bucketName;
+ this.expectedBlobCount = expectedBlobCount;
+ this.deletionWindowSize = Optional.empty();
+ this.associatedProbability = associatedProbability;
+ }
+
+ public Builder deletionWindowSize(int deletionWindowSize) {
+ this.deletionWindowSize = Optional.of(deletionWindowSize);
+ return this;
+ }
+
+ public Builder deletionWindowSize(Optional<Integer> deletionWindowSize) {
+ this.deletionWindowSize = deletionWindowSize;
+ return this;
+ }
+
+ public BlobGCTask build() {
+ return new BlobGCTask(
+ blobStoreDAO,
+ generationAwareBlobIdFactory,
+ generationAwareBlobIdConfiguration,
+ blobReferenceSources,
+ bucketName,
+ clock,
+ expectedBlobCount,
+ deletionWindowSize.orElse(DEFAULT_DELETION_WINDOW_SIZE),
+ associatedProbability);
+ }
}
public static Builder.RequireBlobStoreDAO builder() {
return blobStoreDao -> generationAwareBlobIdFactory -> generationAwareBlobIdConfiguration
-> blobReferenceSources -> bucketName -> clock -> expectedBlobCount
- -> associatedProbability
- -> new BlobGCTask(
- blobStoreDao,
- generationAwareBlobIdFactory,
- generationAwareBlobIdConfiguration,
- blobReferenceSources,
- bucketName,
- clock,
- expectedBlobCount,
- associatedProbability);
+ -> associatedProbability -> new Builder(
+ blobStoreDao,
+ generationAwareBlobIdFactory,
+ generationAwareBlobIdConfiguration,
+ blobReferenceSources,
+ clock,
+ bucketName,
+ expectedBlobCount,
+ associatedProbability);
}
@@ -173,6 +229,7 @@ public class BlobGCTask implements Task {
private final Clock clock;
private final BucketName bucketName;
private final int expectedBlobCount;
+ private final int deletionWindowSize;
private final double associatedProbability;
private final Context context;
@@ -184,7 +241,7 @@ public class BlobGCTask implements Task {
BucketName bucketName,
Clock clock,
int expectedBlobCount,
- double associatedProbability) {
+ int deletionWindowSize, double associatedProbability) {
this.blobStoreDAO = blobStoreDAO;
this.generationAwareBlobIdFactory = generationAwareBlobIdFactory;
this.generationAwareBlobIdConfiguration = generationAwareBlobIdConfiguration;
@@ -192,6 +249,7 @@ public class BlobGCTask implements Task {
this.clock = clock;
this.bucketName = bucketName;
this.expectedBlobCount = expectedBlobCount;
+ this.deletionWindowSize = deletionWindowSize;
this.associatedProbability = associatedProbability;
this.context = new Context(expectedBlobCount, associatedProbability);
}
@@ -205,7 +263,7 @@ public class BlobGCTask implements Task {
generationAwareBlobIdConfiguration,
clock);
- return gcAlgorithm.gc(expectedBlobCount, associatedProbability, bucketName, context)
+ return gcAlgorithm.gc(expectedBlobCount, deletionWindowSize, associatedProbability, bucketName, context)
.block();
}
@@ -216,7 +274,7 @@ public class BlobGCTask implements Task {
@Override
public Optional<TaskExecutionDetails.AdditionalInformation> details() {
- return Optional.of(AdditionalInformation.from(context));
+ return Optional.of(AdditionalInformation.from(context, deletionWindowSize));
}
public Clock getClock() {
@@ -234,4 +292,8 @@ public class BlobGCTask implements Task {
public double getAssociatedProbability() {
return associatedProbability;
}
+
+ public int getDeletionWindowSize() {
+ return deletionWindowSize;
+ }
}
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java
index 8caf790ba6..b379c37b92 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTO.java
@@ -20,6 +20,7 @@
package org.apache.james.server.blob.deduplication;
import java.time.Instant;
+import java.util.Optional;
import org.apache.james.json.DTOModule;
import org.apache.james.server.task.json.dto.AdditionalInformationDTO;
@@ -40,8 +41,8 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
dto.errorCount,
dto.bloomFilterExpectedBlobCount,
dto.bloomFilterAssociatedProbability,
- dto.timestamp
- ))
+ dto.timestamp,
+ dto.deletionWindowSize.orElse(BlobGCTask.Builder.DEFAULT_DELETION_WINDOW_SIZE)))
.toDTOConverter((domain, type) ->
new BlobGCTaskAdditionalInformationDTO(
type,
@@ -51,7 +52,8 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
domain.getGcedBlobCount(),
domain.getErrorCount(),
domain.getBloomFilterExpectedBlobCount(),
- domain.getBloomFilterAssociatedProbability()
+ domain.getBloomFilterAssociatedProbability(),
+ Optional.of(domain.getDeletionWindowSize())
))
.typeName(BlobGCTask.TASK_TYPE.asString())
.withFactory(AdditionalInformationDTOModule::new);
@@ -64,6 +66,7 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
private final long errorCount;
private final long bloomFilterExpectedBlobCount;
private final double bloomFilterAssociatedProbability;
+ private final Optional<Integer> deletionWindowSize;
public BlobGCTaskAdditionalInformationDTO(@JsonProperty("type") String type,
@JsonProperty("timestamp") Instant timestamp,
@@ -72,7 +75,8 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
@JsonProperty("gcedBlobCount") long gcedBlobCount,
@JsonProperty("errorCount") long errorCount,
@JsonProperty("bloomFilterExpectedBlobCount") long bloomFilterExpectedBlobCount,
- @JsonProperty("bloomFilterAssociatedProbability") double bloomFilterAssociatedProbability) {
+ @JsonProperty("bloomFilterAssociatedProbability") double bloomFilterAssociatedProbability,
+ @JsonProperty("deletionWindowSize") Optional<Integer> deletionWindowSize) {
this.type = type;
this.timestamp = timestamp;
this.referenceSourceCount = referenceSourceCount;
@@ -81,6 +85,7 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
this.errorCount = errorCount;
this.bloomFilterExpectedBlobCount = bloomFilterExpectedBlobCount;
this.bloomFilterAssociatedProbability = bloomFilterAssociatedProbability;
+ this.deletionWindowSize = deletionWindowSize;
}
@@ -117,4 +122,8 @@ public class BlobGCTaskAdditionalInformationDTO implements AdditionalInformation
public double getBloomFilterAssociatedProbability() {
return bloomFilterAssociatedProbability;
}
+
+ public Optional<Integer> getDeletionWindowSize() {
+ return deletionWindowSize;
+ }
}
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java
index cdec43f430..2aa7c724fb 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobGCTaskDTO.java
@@ -20,6 +20,7 @@
package org.apache.james.server.blob.deduplication;
import java.time.Clock;
+import java.util.Optional;
import java.util.Set;
import org.apache.james.blob.api.BlobReferenceSource;
@@ -35,15 +36,18 @@ public class BlobGCTaskDTO implements TaskDTO {
private final String bucketName;
private final int expectedBlobCount;
+ private final Optional<Integer> deletionWindowSize;
private final double associatedProbability;
private final String type;
public BlobGCTaskDTO(@JsonProperty("bucketName") String bucketName,
@JsonProperty("expectedBlobCount") int expectedBlobCount,
+ @JsonProperty("deletionWindowSize") Optional<Integer> deletionWindowSize,
@JsonProperty("associatedProbability") double associatedProbability,
@JsonProperty("type") String type) {
this.bucketName = bucketName;
this.expectedBlobCount = expectedBlobCount;
+ this.deletionWindowSize = deletionWindowSize;
this.associatedProbability = associatedProbability;
this.type = type;
}
@@ -64,11 +68,14 @@ public class BlobGCTaskDTO implements TaskDTO {
.bucketName(BucketName.of(dto.bucketName))
.clock(clock)
.expectedBlobCount(dto.expectedBlobCount)
- .associatedProbability(dto.associatedProbability))
+ .associatedProbability(dto.associatedProbability)
+ .deletionWindowSize(dto.deletionWindowSize)
+ .build())
.toDTOConverter((domain, type) ->
new BlobGCTaskDTO(
domain.getBucketName().asString(),
domain.getExpectedBlobCount(),
+ Optional.of(domain.getDeletionWindowSize()),
domain.getAssociatedProbability(),
type))
.typeName(BlobGCTask.TASK_TYPE.asString())
@@ -91,4 +98,8 @@ public class BlobGCTaskDTO implements TaskDTO {
public double getAssociatedProbability() {
return associatedProbability;
}
+
+ public Optional<Integer> getDeletionWindowSize() {
+ return deletionWindowSize;
+ }
}
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
index 841a469ef6..6fbfefbb60 100644
--- a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithm.java
@@ -262,22 +262,22 @@ public class BloomFilterGCAlgorithm {
this.now = clock.instant();
}
- public Mono<Result> gc(int expectedBlobCount, double associatedProbability, BucketName bucketName, Context context) {
+ public Mono<Result> gc(int expectedBlobCount, int deletionWindowSize, double associatedProbability, BucketName bucketName, Context context) {
return populatedBloomFilter(expectedBlobCount, associatedProbability, context)
- .flatMap(bloomFilter -> gc(bloomFilter, bucketName, context))
+ .flatMap(bloomFilter -> gc(bloomFilter, bucketName, context, deletionWindowSize))
.onErrorResume(error -> {
LOGGER.error("Error when running blob deduplicate garbage collection", error);
return Mono.just(Result.PARTIAL);
});
}
- private Mono<Result> gc(BloomFilter<CharSequence> bloomFilter, BucketName bucketName, Context context) {
+ private Mono<Result> gc(BloomFilter<CharSequence> bloomFilter, BucketName bucketName, Context context, int deletionWindowSize) {
return Flux.from(blobStoreDAO.listBlobs(bucketName))
.doOnNext(blobId -> context.incrementBlobCount())
.flatMap(blobId -> Mono.fromCallable(() -> generationAwareBlobIdFactory.from(blobId.asString())))
.filter(blobId -> !blobId.inActiveGeneration(generationAwareBlobIdConfiguration, now))
.filter(blobId -> !bloomFilter.mightContain(salt + blobId.asString()))
- .window(DELETION_BATCH_SIZE)
+ .window(deletionWindowSize)
.flatMap(blobIdFlux -> handlePagedDeletion(bucketName, context, blobIdFlux), DEFAULT_CONCURRENCY)
.reduce(Task::combine)
.switchIfEmpty(Mono.just(Result.COMPLETED));
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java
index e8a6b14a09..34ed5a086f 100644
--- a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskAdditionalInformationDTOTest.java
@@ -19,9 +19,12 @@
package org.apache.james.server.blob.deduplication;
+import static org.assertj.core.api.Assertions.assertThat;
+
import java.time.Instant;
import org.apache.james.JsonSerializationVerifier;
+import org.apache.james.json.JsonGenericSerializer;
import org.apache.james.util.ClassLoaderUtils;
import org.junit.jupiter.api.Test;
@@ -37,9 +40,28 @@ class BlobGCTaskAdditionalInformationDTOTest {
4,
5,
0.8,
- Instant.parse("2007-12-03T10:15:30.00Z")
- ))
+ Instant.parse("2007-12-03T10:15:30.00Z"),
+ 100))
.json(ClassLoaderUtils.getSystemResourceAsString("json/blobGC.additionalInformation.json"))
.verify();
}
+
+ @Test
+ void shouldDeserializeLegacyData() throws Exception {
+ BlobGCTask.AdditionalInformation gcTask = JsonGenericSerializer
+ .forModules(BlobGCTaskAdditionalInformationDTO.SERIALIZATION_MODULE)
+ .withoutNestedType()
+ .deserialize(ClassLoaderUtils.getSystemResourceAsString("json/blobGC-legacy.additionalInformation.json"));
+
+ assertThat(gcTask)
+ .isEqualToComparingFieldByFieldRecursively(new BlobGCTask.AdditionalInformation(
+ 1,
+ 2,
+ 3,
+ 4,
+ 5,
+ 0.8,
+ Instant.parse("2007-12-03T10:15:30.00Z"),
+ 1000));
+ }
}
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java
index 36b4dfaa2e..eb26542b85 100644
--- a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BlobGCTaskSerializationTest.java
@@ -19,6 +19,7 @@
package org.apache.james.server.blob.deduplication;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
import java.time.Clock;
@@ -30,6 +31,7 @@ import org.apache.james.blob.api.BlobReferenceSource;
import org.apache.james.blob.api.BlobStoreDAO;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.json.JsonGenericSerializer;
import org.apache.james.util.ClassLoaderUtils;
import org.apache.james.utils.UpdatableTickingClock;
import org.junit.jupiter.api.BeforeEach;
@@ -69,9 +71,36 @@ class BlobGCTaskSerializationTest {
BucketName.DEFAULT,
clock,
99,
+ 100,
0.8
))
.json(ClassLoaderUtils.getSystemResourceAsString("json/blobGC.task.json"))
.verify();
}
+
+ @Test
+ void shouldDeserializeLegacyData() throws Exception {
+ BlobGCTask gcTask = JsonGenericSerializer
+ .forModules(BlobGCTaskDTO.module(
+ blobStoreDAO,
+ generationAwareBlobIdFactory,
+ generationAwareBlobIdConfiguration,
+ blobReferenceSources,
+ clock))
+ .withoutNestedType()
+ .deserialize(ClassLoaderUtils.getSystemResourceAsString("json/blobGC-legacy.task.json"));
+
+ assertThat(gcTask)
+ .isEqualToComparingFieldByFieldRecursively(new BlobGCTask(
+ blobStoreDAO,
+ generationAwareBlobIdFactory,
+ generationAwareBlobIdConfiguration,
+ blobReferenceSources,
+ BucketName.DEFAULT,
+ clock,
+ 99,
+ 1000,
+ 0.8
+ ));
+ }
}
diff --git a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
index c300cb50be..9cf30eec9f 100644
--- a/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
+++ b/server/blob/blob-storage-strategy/src/test/java/org/apache/james/server/blob/deduplication/BloomFilterGCAlgorithmContract.java
@@ -62,6 +62,8 @@ public interface BloomFilterGCAlgorithmContract {
BucketName DEFAULT_BUCKET = BucketName.of("default");
GenerationAwareBlobId.Configuration GENERATION_AWARE_BLOB_ID_CONFIGURATION = GenerationAwareBlobId.Configuration.DEFAULT;
int EXPECTED_BLOB_COUNT = 100;
+ int DELETION_WINDOW_SIZE = 10;
+
double ASSOCIATED_PROBABILITY = 0.01;
ConditionFactory CALMLY_AWAIT = Awaitility
@@ -102,7 +104,7 @@ public interface BloomFilterGCAlgorithmContract {
Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
BloomFilterGCAlgorithm bloomFilterGCAlgorithm = bloomFilterGCAlgorithm();
- Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+ Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
assertThat(result).isEqualTo(Task.Result.COMPLETED);
assertThat(context.snapshot())
@@ -126,7 +128,7 @@ public interface BloomFilterGCAlgorithmContract {
Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
BloomFilterGCAlgorithm bloomFilterGCAlgorithm = bloomFilterGCAlgorithm();
- Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+ Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
assertThat(result).isEqualTo(Task.Result.COMPLETED);
assertThat(context.snapshot())
@@ -151,7 +153,7 @@ public interface BloomFilterGCAlgorithmContract {
Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
BloomFilterGCAlgorithm bloomFilterGCAlgorithm = bloomFilterGCAlgorithm();
- Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+ Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
assertThat(result).isEqualTo(Task.Result.COMPLETED);
assertThat(context.snapshot())
@@ -182,7 +184,7 @@ public interface BloomFilterGCAlgorithmContract {
Context context = new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY);
BloomFilterGCAlgorithm bloomFilterGCAlgorithm = bloomFilterGCAlgorithm();
- Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+ Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
assertThat(result).isEqualTo(Task.Result.COMPLETED);
Context.Snapshot snapshot = context.snapshot();
@@ -217,6 +219,7 @@ public interface BloomFilterGCAlgorithmContract {
CALMLY_AWAIT.untilAsserted(() -> {
Mono.from(bloomFilterGCAlgorithm().gc(
EXPECTED_BLOB_COUNT,
+ DELETION_WINDOW_SIZE,
ASSOCIATED_PROBABILITY,
DEFAULT_BUCKET,
new Context(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY)))
@@ -245,7 +248,7 @@ public interface BloomFilterGCAlgorithmContract {
GENERATION_AWARE_BLOB_ID_FACTORY,
GENERATION_AWARE_BLOB_ID_CONFIGURATION,
CLOCK);
- Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
+ Task.Result result = Mono.from(bloomFilterGCAlgorithm.gc(EXPECTED_BLOB_COUNT, DELETION_WINDOW_SIZE, ASSOCIATED_PROBABILITY, DEFAULT_BUCKET, context)).block();
assertThat(result).isEqualTo(Task.Result.PARTIAL);
assertThat(context.snapshot())
diff --git a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC-legacy.additionalInformation.json
similarity index 100%
copy from server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json
copy to server/blob/blob-storage-strategy/src/test/resources/json/blobGC-legacy.additionalInformation.json
diff --git a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC-legacy.task.json
similarity index 100%
copy from server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json
copy to server/blob/blob-storage-strategy/src/test/resources/json/blobGC-legacy.task.json
diff --git a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json
index ba4ddee2b5..a669c2edd2 100644
--- a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json
+++ b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.additionalInformation.json
@@ -6,5 +6,6 @@
"gcedBlobCount": 3,
"errorCount": 4,
"bloomFilterExpectedBlobCount": 5,
- "bloomFilterAssociatedProbability": 0.8
+ "bloomFilterAssociatedProbability": 0.8,
+ "deletionWindowSize": 100
}
\ No newline at end of file
diff --git a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json
index 44f1dac469..f07977fe60 100644
--- a/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json
+++ b/server/blob/blob-storage-strategy/src/test/resources/json/blobGC.task.json
@@ -2,5 +2,6 @@
"associatedProbability": 0.8,
"bucketName": "default",
"expectedBlobCount": 99,
+ "deletionWindowSize": 100,
"type": "BlobGCTask"
}
\ No newline at end of file
diff --git a/server/protocols/webadmin/webadmin-data/src/main/java/org/apache/james/webadmin/routes/BlobRoutes.java b/server/protocols/webadmin/webadmin-data/src/main/java/org/apache/james/webadmin/routes/BlobRoutes.java
index 6b50486ec5..6bdb0cac3b 100644
--- a/server/protocols/webadmin/webadmin-data/src/main/java/org/apache/james/webadmin/routes/BlobRoutes.java
+++ b/server/protocols/webadmin/webadmin-data/src/main/java/org/apache/james/webadmin/routes/BlobRoutes.java
@@ -95,6 +95,7 @@ public class BlobRoutes implements Routes {
"'scope' is missing or must be 'unreferenced'");
int expectedBlobCount = getExpectedBlobCount(request).orElse(EXPECTED_BLOB_COUNT_DEFAULT);
+ Optional<Integer> deletionWindowSize = getDeletionWindowSize(request);
double associatedProbability = getAssociatedProbability(request).orElse(ASSOCIATED_PROBABILITY_DEFAULT);
return BlobGCTask.builder()
@@ -105,7 +106,9 @@ public class BlobRoutes implements Routes {
.bucketName(bucketName)
.clock(clock)
.expectedBlobCount(expectedBlobCount)
- .associatedProbability(associatedProbability);
+ .associatedProbability(associatedProbability)
+ .deletionWindowSize(deletionWindowSize)
+ .build();
}
private static Optional<Integer> getExpectedBlobCount(Request req) {
@@ -122,6 +125,20 @@ public class BlobRoutes implements Routes {
}
}
+ private static Optional<Integer> getDeletionWindowSize(Request req) {
+ try {
+ return Optional.ofNullable(req.queryParams("deletionWindowSize"))
+ .map(Integer::parseInt)
+ .map(expectedBlobCount -> {
+ Preconditions.checkArgument(expectedBlobCount > 0,
+ "'deletionWindowSize' must be strictly positive");
+ return expectedBlobCount;
+ });
+ } catch (NumberFormatException ex) {
+ throw new IllegalArgumentException("'deletionWindowSize' must be numeric");
+ }
+ }
+
private static Optional<Double> getAssociatedProbability(Request req) {
try {
return Optional.ofNullable(req.queryParams("associatedProbability"))
diff --git a/server/protocols/webadmin/webadmin-data/src/test/java/org/apache/james/webadmin/routes/BlobRoutesTest.java b/server/protocols/webadmin/webadmin-data/src/test/java/org/apache/james/webadmin/routes/BlobRoutesTest.java
index f04bbae2ff..f612fab960 100644
--- a/server/protocols/webadmin/webadmin-data/src/test/java/org/apache/james/webadmin/routes/BlobRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-data/src/test/java/org/apache/james/webadmin/routes/BlobRoutesTest.java
@@ -189,6 +189,7 @@ class BlobRoutesTest {
.body("additionalInformation.blobCount", is(0))
.body("additionalInformation.gcedBlobCount", is(0))
.body("additionalInformation.errorCount", is(0))
+ .body("additionalInformation.deletionWindowSize", is(1000))
.body("additionalInformation.bloomFilterExpectedBlobCount", is(1_000_000))
.body("additionalInformation.bloomFilterAssociatedProbability", is(0.01F));
}
@@ -210,6 +211,23 @@ class BlobRoutesTest {
.body("additionalInformation.bloomFilterExpectedBlobCount", is(99));
}
+ @Test
+ void deleteUnReferencedShouldAcceptDeletionWindowSizeParam() {
+ String taskId = given()
+ .queryParam("scope", "unreferenced")
+ .queryParam("deletionWindowSize", 99)
+ .delete()
+ .jsonPath()
+ .get("taskId");
+
+ given()
+ .basePath(TasksRoutes.BASE)
+ .when()
+ .get(taskId + "/await")
+ .then()
+ .body("additionalInformation.deletionWindowSize", is(99));
+ }
+
@ParameterizedTest
@MethodSource("expectedBlobCountParameters")
void deleteUnReferencedShouldReturnErrorWhenExpectedBlobCountInvalid(Object expectedBlobCount) {
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org