You are viewing a plain text version of this content. The canonical link for it is here.
Posted to server-dev@james.apache.org by bt...@apache.org on 2020/07/24 10:34:31 UTC
[james-project] branch master updated (37b2d0b -> 26268b1)
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git.
from 37b2d0b JAMES-3313 Lazily instanciate BlobStore within mailbox/cassandra tests
new 649b611 JAMES-3314 rename module blob-deduplicating to blob-storage-strategy
new 5e27563 JAMES-3314 implement Passtrough blobstore
new 700116f7 JAMES-3314 Mutualize CassandraBlobStoreContract
new e2c8fc2 JAMES-3314 Avoid double MetricableBlobStore wrapping in MemoryBlobStoreTest
new 7e8e2be JAMES-3314 Extract test enforcing content-based BlobId generation
new ddd3be5 JAMES-3314 Rewrite DeletedMessagesVaultRoutesTest so that it do not hard code BlobId generation logic
new 26268b1 JAMES-3224 Use CassandraConsistenciesConfiguration for default consistency levels
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../backends/cassandra/init/ClusterFactory.java | 11 +--
.../cassandra/init/ResilientClusterProvider.java | 9 +-
.../CassandraConsistenciesConfiguration.java | 2 +
.../james/backends/cassandra/CassandraCluster.java | 3 +-
.../james/backends/cassandra/DockerCassandra.java | 20 +----
.../cassandra/init/ClusterFactoryTest.java | 7 +-
.../init/ResilientClusterProviderTest.java | 8 +-
.../SessionWithInitializedTablesFactoryTest.java | 3 +-
.../james/vault/DeletedMessageVaultHookTest.java | 5 +-
.../blob/BlobStoreDeletedMessageVaultTest.java | 5 +-
pom.xml | 2 +-
.../apache/james/blob/api/BlobStoreContract.java | 33 --------
.../blob/api/DeduplicationBlobStoreContract.java | 82 ++++++++++++++++++
.../james/blob/api/DeleteBlobStoreContract.java | 8 +-
server/blob/blob-cassandra/pom.xml | 2 +-
.../blob/cassandra/CassandraBlobStoreFactory.java | 17 ++--
...reTest.java => CassandraBlobStoreContract.java} | 82 +++++-------------
.../blob/cassandra/CassandraBlobStoreTest.java | 99 +++-------------------
....java => CassandraPassTroughBlobStoreTest.java} | 45 ++++++----
.../file/LocalFileBlobExportMechanismTest.java | 5 +-
server/blob/blob-memory/pom.xml | 2 +-
.../james/blob/memory/MemoryBlobStoreFactory.java | 15 +---
...st.java => MemoryBlobStorePassThroughTest.java} | 7 +-
.../james/blob/memory/MemoryBlobStoreTest.java | 9 +-
.../ObjectStorageBlobStoreAWSCryptoTest.java | 3 +-
.../ObjectStorageBlobStoreAWSNamespaceTest.java | 3 +-
...tStorageBlobStoreAWSPrefixAndNamespaceTest.java | 3 +-
.../ObjectStorageBlobStoreAWSPrefixTest.java | 3 +-
.../ObjectStorageBlobStoreAWSTest.java | 3 +-
.../objectstorage/ObjectStorageBlobStoreTest.java | 3 +-
.../pom.xml | 2 +-
.../blob/deduplication/BlobStoreFactory.java | 72 ++++++++++++++++
.../server/blob/deduplication/StorageStrategy.java | 7 +-
.../deduplication/DeDuplicationBlobStore.scala | 0
.../blob/deduplication/PassThroughBlobStore.scala} | 38 ++-------
.../james/blob/mail/MimeMessageStoreTest.java | 5 +-
server/blob/pom.xml | 2 +-
.../rabbitmq/FixingGhostMailboxTest.java | 3 +-
.../routes/DeletedMessagesVaultRoutesTest.java | 24 ++++--
.../james/webadmin/service/ExportServiceTest.java | 5 --
.../webadmin/service/ExportServiceTestSystem.java | 5 +-
.../linshare/LinshareBlobExportMechanismTest.java | 5 +-
42 files changed, 339 insertions(+), 328 deletions(-)
create mode 100644 server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java
copy server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/{CassandraBlobStoreTest.java => CassandraBlobStoreContract.java} (51%)
copy server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/{CassandraDumbBlobStoreTest.java => CassandraPassTroughBlobStoreTest.java} (62%)
copy server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/{MemoryBlobStoreTest.java => MemoryBlobStorePassThroughTest.java} (88%)
rename server/blob/{blob-deduplicating => blob-storage-strategy}/pom.xml (98%)
create mode 100644 server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobStoreFactory.java
copy json/src/test/java/org/apache/james/dto/BaseType.java => server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/StorageStrategy.java (91%)
copy server/blob/{blob-deduplicating => blob-storage-strategy}/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala (100%)
rename server/blob/{blob-deduplicating/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala => blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala} (61%)
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 01/07: JAMES-3314 rename module blob-deduplicating
to blob-storage-strategy
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 649b611f6ff3166284b87b19d8d28b9b0d7b2924
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Tue Jul 21 11:02:23 2020 +0200
JAMES-3314 rename module blob-deduplicating to blob-storage-strategy
---
pom.xml | 2 +-
server/blob/blob-cassandra/pom.xml | 2 +-
server/blob/blob-memory/pom.xml | 2 +-
.../memory/MemoryBlobStorePassThroughTest.java | 48 ++++++++++++++++++++++
.../pom.xml | 2 +-
.../deduplication/DeDuplicationBlobStore.scala | 0
.../blob/deduplication/PassThroughBlobStore.scala} | 0
server/blob/pom.xml | 2 +-
8 files changed, 53 insertions(+), 5 deletions(-)
diff --git a/pom.xml b/pom.xml
index fbdf065..54cf223 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1128,7 +1128,7 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>blob-deduplicating</artifactId>
+ <artifactId>blob-storage-strategy</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
diff --git a/server/blob/blob-cassandra/pom.xml b/server/blob/blob-cassandra/pom.xml
index 733596a..6f6e03e 100644
--- a/server/blob/blob-cassandra/pom.xml
+++ b/server/blob/blob-cassandra/pom.xml
@@ -55,7 +55,7 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>blob-deduplicating</artifactId>
+ <artifactId>blob-storage-strategy</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
diff --git a/server/blob/blob-memory/pom.xml b/server/blob/blob-memory/pom.xml
index 4224387..80e6f6a 100644
--- a/server/blob/blob-memory/pom.xml
+++ b/server/blob/blob-memory/pom.xml
@@ -44,7 +44,7 @@
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
- <artifactId>blob-deduplicating</artifactId>
+ <artifactId>blob-storage-strategy</artifactId>
</dependency>
<dependency>
<groupId>${james.groupId}</groupId>
diff --git a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java
new file mode 100644
index 0000000..80dba7e
--- /dev/null
+++ b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java
@@ -0,0 +1,48 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.memory;
+
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.HashBlobId;
+import org.apache.james.blob.api.MetricableBlobStore;
+import org.apache.james.blob.api.MetricableBlobStoreContract;
+import org.junit.jupiter.api.BeforeEach;
+
+public class MemoryBlobStoreTest implements MetricableBlobStoreContract {
+
+ private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
+ private BlobStore blobStore;
+
+ @BeforeEach
+ void setUp() {
+ blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.create(BLOB_ID_FACTORY));
+ }
+
+ @Override
+ public BlobStore testee() {
+ return blobStore;
+ }
+
+ @Override
+ public BlobId.Factory blobIdFactory() {
+ return BLOB_ID_FACTORY;
+ }
+}
diff --git a/server/blob/blob-deduplicating/pom.xml b/server/blob/blob-storage-strategy/pom.xml
similarity index 98%
rename from server/blob/blob-deduplicating/pom.xml
rename to server/blob/blob-storage-strategy/pom.xml
index b4d2740..e2ba795 100644
--- a/server/blob/blob-deduplicating/pom.xml
+++ b/server/blob/blob-storage-strategy/pom.xml
@@ -27,7 +27,7 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>blob-deduplicating</artifactId>
+ <artifactId>blob-storage-strategy</artifactId>
<packaging>jar</packaging>
<name>Apache James :: Server :: Blob :: Deduplicating Blob Storage</name>
diff --git a/server/blob/blob-deduplicating/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
similarity index 100%
copy from server/blob/blob-deduplicating/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
copy to server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
diff --git a/server/blob/blob-deduplicating/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
similarity index 100%
rename from server/blob/blob-deduplicating/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
rename to server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
diff --git a/server/blob/pom.xml b/server/blob/pom.xml
index a4d6742..14e37f3 100644
--- a/server/blob/pom.xml
+++ b/server/blob/pom.xml
@@ -34,7 +34,7 @@
<modules>
<module>blob-api</module>
- <module>blob-deduplicating</module>
+ <module>blob-storage-strategy</module>
<module>blob-cassandra</module>
<module>blob-common</module>
<module>blob-export-api</module>
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 04/07: JAMES-3314 Avoid double MetricableBlobStore
wrapping in MemoryBlobStoreTest
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit e2c8fc2dba126baf5d89da79a81ebcfe32fbc85c
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Jul 22 12:00:42 2020 +0700
JAMES-3314 Avoid double MetricableBlobStore wrapping in MemoryBlobStoreTest
---
.../java/org/apache/james/blob/memory/MemoryBlobStoreTest.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
index 147133c..a2ef603 100644
--- a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
+++ b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
@@ -33,10 +33,10 @@ public class MemoryBlobStoreTest implements MetricableBlobStoreContract {
@BeforeEach
void setUp() {
- blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.builder()
- .blobIdFactory(BLOB_ID_FACTORY)
- .defaultBucketName()
- .deduplication()));
+ blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.builder()
+ .blobIdFactory(BLOB_ID_FACTORY)
+ .defaultBucketName()
+ .deduplication());
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 05/07: JAMES-3314 Extract test enforcing
content-based BlobId generation
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 7e8e2bea9d8b82d24b5624b43d67473a361b7e23
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Jul 22 12:03:19 2020 +0700
JAMES-3314 Extract test enforcing content-based BlobId generation
Some tests can be adapted to not rely on content based BlobId generation, others should be moved to a separated contract
---
.../apache/james/blob/api/BlobStoreContract.java | 33 ---------
.../blob/api/DeduplicationBlobStoreContract.java | 82 ++++++++++++++++++++++
.../james/blob/api/DeleteBlobStoreContract.java | 8 +--
.../blob/cassandra/CassandraBlobStoreTest.java | 3 +-
.../james/blob/memory/MemoryBlobStoreTest.java | 4 +-
.../ObjectStorageBlobStoreAWSCryptoTest.java | 3 +-
.../ObjectStorageBlobStoreAWSNamespaceTest.java | 3 +-
...tStorageBlobStoreAWSPrefixAndNamespaceTest.java | 3 +-
.../ObjectStorageBlobStoreAWSPrefixTest.java | 3 +-
.../ObjectStorageBlobStoreAWSTest.java | 3 +-
.../objectstorage/ObjectStorageBlobStoreTest.java | 3 +-
11 files changed, 102 insertions(+), 46 deletions(-)
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
index dc49969..b9f8e1b 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
@@ -127,39 +127,6 @@ public interface BlobStoreContract extends DeleteBlobStoreContract, BucketBlobSt
assertThat(new String(bytes, StandardCharsets.UTF_8)).isEmpty();
}
- @ParameterizedTest
- @MethodSource("storagePolicies")
- default void saveShouldReturnBlobId(BlobStore.StoragePolicy storagePolicy) {
- BlobStore store = testee();
- BucketName defaultBucketName = store.getDefaultBucketName();
-
- BlobId blobId = Mono.from(store.save(defaultBucketName, SHORT_BYTEARRAY, storagePolicy)).block();
-
- assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
- }
-
- @ParameterizedTest
- @MethodSource("storagePolicies")
- default void saveShouldReturnBlobIdOfString(BlobStore.StoragePolicy storagePolicy) {
- BlobStore store = testee();
- BucketName defaultBucketName = store.getDefaultBucketName();
-
- BlobId blobId = Mono.from(store.save(defaultBucketName, SHORT_STRING, storagePolicy)).block();
-
- assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
- }
-
- @ParameterizedTest
- @MethodSource("storagePolicies")
- default void saveShouldReturnBlobIdOfInputStream(BlobStore.StoragePolicy storagePolicy) {
- BlobStore store = testee();
- BucketName defaultBucketName = store.getDefaultBucketName();
-
- BlobId blobId = Mono.from(store.save(defaultBucketName, new ByteArrayInputStream(SHORT_BYTEARRAY), storagePolicy)).block();
-
- assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
- }
-
@Test
default void readBytesShouldThrowWhenNoExisting() {
BlobStore store = testee();
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java
new file mode 100644
index 0000000..b92e805
--- /dev/null
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeduplicationBlobStoreContract.java
@@ -0,0 +1,82 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.api;
+
+import static org.apache.james.blob.api.BlobStore.StoragePolicy.HIGH_PERFORMANCE;
+import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST;
+import static org.apache.james.blob.api.BlobStore.StoragePolicy.SIZE_BASED;
+import static org.apache.james.blob.api.BlobStoreContract.SHORT_BYTEARRAY;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.io.ByteArrayInputStream;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import reactor.core.publisher.Mono;
+
+public interface DeduplicationBlobStoreContract {
+ static Stream<Arguments> storagePolicies() {
+ return Stream.of(
+ Arguments.arguments(LOW_COST),
+ Arguments.arguments(SIZE_BASED),
+ Arguments.arguments(HIGH_PERFORMANCE));
+ }
+ String SHORT_STRING = "toto";
+
+ BlobStore testee();
+
+ BlobId.Factory blobIdFactory();
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
+ default void saveShouldReturnBlobIdOfString(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, SHORT_STRING, storagePolicy)).block();
+
+ assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
+ default void saveShouldReturnBlobId(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, SHORT_BYTEARRAY, storagePolicy)).block();
+
+ assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
+ default void saveShouldReturnBlobIdOfInputStream(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, new ByteArrayInputStream(SHORT_BYTEARRAY), storagePolicy)).block();
+
+ assertThat(blobId).isEqualTo(blobIdFactory().from("31f7a65e315586ac198bd798b6629ce4903d0899476d5741a9f32e2e521b6a66"));
+ }
+}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeleteBlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeleteBlobStoreContract.java
index a65362a..afc65f5 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeleteBlobStoreContract.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/DeleteBlobStoreContract.java
@@ -139,12 +139,12 @@ public interface DeleteBlobStoreContract {
BlobStore store = testee();
BucketName defaultBucketName = store.getDefaultBucketName();
- Mono.from(store.save(CUSTOM, SHORT_BYTEARRAY, LOW_COST)).block();
- BlobId blobId = Mono.from(store.save(defaultBucketName, SHORT_BYTEARRAY, LOW_COST)).block();
+ BlobId blobIdInCustomBucket = Mono.from(store.save(CUSTOM, SHORT_BYTEARRAY, LOW_COST)).block();
+ BlobId blobIdInDefaultBucket = Mono.from(store.save(defaultBucketName, SHORT_BYTEARRAY, LOW_COST)).block();
- Mono.from(store.delete(defaultBucketName, blobId)).block();
+ Mono.from(store.delete(defaultBucketName, blobIdInDefaultBucket)).block();
- InputStream read = store.read(CUSTOM, blobId);
+ InputStream read = store.read(CUSTOM, blobIdInCustomBucket);
assertThat(read).hasSameContentAs(new ByteArrayInputStream(SHORT_BYTEARRAY));
}
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
index 73b031a..56e6ac0 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
@@ -27,13 +27,14 @@ import org.apache.james.backends.cassandra.init.configuration.CassandraConfigura
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.extension.RegisterExtension;
-public class CassandraBlobStoreTest implements CassandraBlobStoreContract {
+public class CassandraBlobStoreTest implements CassandraBlobStoreContract, DeduplicationBlobStoreContract {
@RegisterExtension
static CassandraClusterExtension cassandraCluster = new CassandraClusterExtension(CassandraBlobModule.MODULE);
diff --git a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
index a2ef603..f678066 100644
--- a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
+++ b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
@@ -21,13 +21,13 @@ package org.apache.james.blob.memory;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
import org.junit.jupiter.api.BeforeEach;
-public class MemoryBlobStoreTest implements MetricableBlobStoreContract {
-
+public class MemoryBlobStoreTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
private BlobStore blobStore;
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSCryptoTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSCryptoTest.java
index aba7d13..3a23b22 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSCryptoTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSCryptoTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
@@ -38,7 +39,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-public class ObjectStorageBlobStoreAWSCryptoTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobStoreAWSCryptoTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
private static final Credentials PASSWORD = Credentials.of("testing");
private static final String SAMPLE_SALT = "c603a7327ee3dcbc031d8d34b1096c605feca5e1";
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSNamespaceTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSNamespaceTest.java
index fdf697b..9b17ceb 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSNamespaceTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSNamespaceTest.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
@@ -37,7 +38,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-public class ObjectStorageBlobStoreAWSNamespaceTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobStoreAWSNamespaceTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
private BlobStore testee;
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixAndNamespaceTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixAndNamespaceTest.java
index aea96a0..4592820 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixAndNamespaceTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixAndNamespaceTest.java
@@ -24,6 +24,7 @@ import java.io.IOException;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
@@ -37,7 +38,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-public class ObjectStorageBlobStoreAWSPrefixAndNamespaceTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobStoreAWSPrefixAndNamespaceTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
private BlobStore testee;
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixTest.java
index f82ed8f..3de820c 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSPrefixTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
@@ -36,7 +37,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-public class ObjectStorageBlobStoreAWSPrefixTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobStoreAWSPrefixTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
private BlobStore testee;
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSTest.java
index 6af3fd3..1e66e98 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreAWSTest.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
@@ -36,7 +37,7 @@ import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.extension.ExtendWith;
@ExtendWith(DockerAwsS3Extension.class)
-public class ObjectStorageBlobStoreAWSTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobStoreAWSTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
diff --git a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreTest.java b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreTest.java
index 10fb3c9..2505b39 100644
--- a/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreTest.java
+++ b/server/blob/blob-objectstorage/src/test/java/org/apache/james/blob/objectstorage/ObjectStorageBlobStoreTest.java
@@ -31,6 +31,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.james.blob.api.BlobId;
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DeduplicationBlobStoreContract;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
@@ -56,7 +57,7 @@ import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
@ExtendWith(DockerSwiftExtension.class)
-public class ObjectStorageBlobStoreTest implements MetricableBlobStoreContract {
+public class ObjectStorageBlobStoreTest implements MetricableBlobStoreContract, DeduplicationBlobStoreContract {
private static final String BIG_STRING = Strings.repeat("big blob content", 10 * 1024);
private static final TenantName TENANT_NAME = TenantName.of("test");
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 06/07: JAMES-3314 Rewrite
DeletedMessagesVaultRoutesTest so that it do not hard code BlobId
generation logic
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit ddd3be5fcd5d754be9c671bc5bd5263937bbd844
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Thu Jul 23 09:06:20 2020 +0700
JAMES-3314 Rewrite DeletedMessagesVaultRoutesTest so that it do not hard code BlobId generation logic
---
.../vault/routes/DeletedMessagesVaultRoutesTest.java | 19 ++++++++++++-------
1 file changed, 12 insertions(+), 7 deletions(-)
diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
index 3cf3d00..1c8d463 100644
--- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
@@ -138,19 +138,24 @@ import reactor.core.publisher.Mono;
class DeletedMessagesVaultRoutesTest {
- private class NoopBlobExporting implements BlobExportMechanism {
+ private static class NoopBlobExporting implements BlobExportMechanism {
+ private Optional<BlobId> exportedBlobId = Optional.empty();
+
@Override
public ShareeStage blobId(BlobId blobId) {
- return exportTo -> explanation -> fileCustomPrefix -> fileExtension -> () -> export(exportTo, explanation);
+ return exportTo -> explanation -> fileCustomPrefix -> fileExtension -> () -> export(exportTo, explanation, blobId);
+ }
+
+ void export(MailAddress exportTo, String explanation, BlobId blobId) {
+ this.exportedBlobId = Optional.of(blobId);
}
- void export(MailAddress exportTo, String explanation) {
- // do nothing
+ public Optional<BlobId> getExportedBlobId() {
+ return exportedBlobId;
}
}
private static final ZonedDateTime NOW = ZonedDateTime.parse("2015-10-30T16:12:00Z");
- private static final ZonedDateTime OLD_DELIVERY_DATE = ZonedDateTime.parse("2010-10-30T14:12:00Z");
private static final ZonedDateTime OLD_DELETION_DATE = ZonedDateTime.parse("2010-10-30T15:12:00Z");
private static final String MATCH_ALL_QUERY = "{" +
"\"combinator\": \"and\"," +
@@ -1891,7 +1896,7 @@ class DeletedMessagesVaultRoutesTest {
.get(taskId + "/await");
verify(blobExporting, times(1))
- .export(eq(USERNAME_2.asMailAddress()), any());
+ .export(eq(USERNAME_2.asMailAddress()), any(), any());
}
@Test
@@ -1935,7 +1940,7 @@ class DeletedMessagesVaultRoutesTest {
byte[] expectedZippedData = zippedMessagesData();
- assertThat(blobStore.read(blobStore.getDefaultBucketName(), blobIdFactory.forPayload(expectedZippedData)))
+ assertThat(blobStore.read(blobStore.getDefaultBucketName(), blobExporting.getExportedBlobId().get()))
.hasSameContentAs(new ByteArrayInputStream(expectedZippedData));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 07/07: JAMES-3224 Use
CassandraConsistenciesConfiguration for default consistency levels
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 26268b1817e4af7f9cc788b85e80f08ae977f6b1
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Jul 22 15:20:26 2020 +0700
JAMES-3224 Use CassandraConsistenciesConfiguration for default consistency levels
---
.../backends/cassandra/init/ClusterFactory.java | 11 ++++++-----
.../cassandra/init/ResilientClusterProvider.java | 9 +++++----
.../CassandraConsistenciesConfiguration.java | 2 ++
.../james/backends/cassandra/CassandraCluster.java | 3 ++-
.../james/backends/cassandra/DockerCassandra.java | 20 +++-----------------
.../backends/cassandra/init/ClusterFactoryTest.java | 7 ++++---
.../cassandra/init/ResilientClusterProviderTest.java | 8 +++++---
.../SessionWithInitializedTablesFactoryTest.java | 3 ++-
.../integration/rabbitmq/FixingGhostMailboxTest.java | 3 ++-
9 files changed, 31 insertions(+), 35 deletions(-)
diff --git a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java
index dc696df..30bfa56 100644
--- a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java
+++ b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ClusterFactory.java
@@ -21,11 +21,11 @@ package org.apache.james.backends.cassandra.init;
import static com.datastax.driver.core.querybuilder.QueryBuilder.select;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import com.datastax.driver.core.BoundStatement;
import com.datastax.driver.core.Cluster;
-import com.datastax.driver.core.ConsistencyLevel;
import com.datastax.driver.core.QueryOptions;
import com.datastax.driver.core.Session;
import com.datastax.driver.core.SocketOptions;
@@ -33,7 +33,7 @@ import com.google.common.base.Preconditions;
public class ClusterFactory {
- public static Cluster create(ClusterConfiguration configuration) {
+ public static Cluster create(ClusterConfiguration configuration, CassandraConsistenciesConfiguration consistenciesConfiguration) {
Preconditions.checkState(configuration.getUsername().isPresent() == configuration.getPassword().isPresent(), "If you specify username, you must specify password");
Cluster.Builder clusterBuilder = Cluster.builder()
@@ -46,7 +46,7 @@ public class ClusterFactory {
configuration.getPassword().ifPresent(password ->
clusterBuilder.withCredentials(username, password)));
- clusterBuilder.withQueryOptions(queryOptions());
+ clusterBuilder.withQueryOptions(queryOptions(consistenciesConfiguration));
SocketOptions socketOptions = new SocketOptions();
socketOptions.setReadTimeoutMillis(configuration.getReadTimeoutMillis());
@@ -71,9 +71,10 @@ public class ClusterFactory {
}
}
- private static QueryOptions queryOptions() {
+ private static QueryOptions queryOptions(CassandraConsistenciesConfiguration consistenciesConfiguration) {
return new QueryOptions()
- .setConsistencyLevel(ConsistencyLevel.QUORUM);
+ .setConsistencyLevel(consistenciesConfiguration.getRegular())
+ .setSerialConsistencyLevel(consistenciesConfiguration.getLightweightTransaction());
}
private static void ensureContactable(Cluster cluster) {
diff --git a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ResilientClusterProvider.java b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ResilientClusterProvider.java
index 4eeb502..999e24d 100644
--- a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ResilientClusterProvider.java
+++ b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/ResilientClusterProvider.java
@@ -30,6 +30,7 @@ import javax.inject.Inject;
import javax.inject.Provider;
import javax.inject.Singleton;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,21 +51,21 @@ public class ResilientClusterProvider implements Provider<Cluster> {
@VisibleForTesting
@Inject
- ResilientClusterProvider(ClusterConfiguration configuration) {
+ ResilientClusterProvider(ClusterConfiguration configuration, CassandraConsistenciesConfiguration consistenciesConfiguration) {
Duration waitDelay = Duration.ofMillis(configuration.getMinDelay());
- cluster = Mono.fromCallable(getClusterRetryCallable(configuration))
+ cluster = Mono.fromCallable(getClusterRetryCallable(configuration, consistenciesConfiguration))
.doOnError(e -> LOGGER.warn("Error establishing Cassandra connection. Next retry scheduled in {} ms", waitDelay, e))
.retryWhen(Retry.backoff(configuration.getMaxRetry(), waitDelay).scheduler(Schedulers.elastic()))
.publishOn(Schedulers.elastic())
.block();
}
- private Callable<Cluster> getClusterRetryCallable(ClusterConfiguration configuration) {
+ private Callable<Cluster> getClusterRetryCallable(ClusterConfiguration configuration, CassandraConsistenciesConfiguration consistenciesConfiguration) {
LOGGER.info("Trying to connect to Cassandra service at {} (list {})", LocalDateTime.now(),
ImmutableList.copyOf(configuration.getHosts()).toString());
return () -> {
- Cluster cluster = ClusterFactory.create(configuration);
+ Cluster cluster = ClusterFactory.create(configuration, consistenciesConfiguration);
try {
keyspaceExist(cluster, "any"); // plays a sample query to ensure we can contact the cluster
return cluster;
diff --git a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/configuration/CassandraConsistenciesConfiguration.java b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/configuration/CassandraConsistenciesConfiguration.java
index e1f71e6..8a70b45 100644
--- a/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/configuration/CassandraConsistenciesConfiguration.java
+++ b/backends-common/cassandra/src/main/java/org/apache/james/backends/cassandra/init/configuration/CassandraConsistenciesConfiguration.java
@@ -25,6 +25,8 @@ import com.datastax.driver.core.ConsistencyLevel;
import com.google.common.base.MoreObjects;
public class CassandraConsistenciesConfiguration {
+ public static final CassandraConsistenciesConfiguration DEFAULT = new CassandraConsistenciesConfiguration(ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL);
+
public static ConsistencyLevel fromString(String value) {
switch (value) {
case "QUORUM":
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/CassandraCluster.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/CassandraCluster.java
index 2090236..48c787f 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/CassandraCluster.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/CassandraCluster.java
@@ -25,6 +25,7 @@ import org.apache.james.backends.cassandra.init.CassandraTableManager;
import org.apache.james.backends.cassandra.init.CassandraTypesProvider;
import org.apache.james.backends.cassandra.init.ClusterFactory;
import org.apache.james.backends.cassandra.init.SessionWithInitializedTablesFactory;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import org.apache.james.backends.cassandra.init.configuration.KeyspaceConfiguration;
import org.apache.james.util.Host;
@@ -59,7 +60,7 @@ public final class CassandraCluster implements AutoCloseable {
this.module = module;
this.clusterConfiguration = DockerCassandra.configurationBuilder(host).build();
- this.nonPrivilegedCluster = ClusterFactory.create(clusterConfiguration);
+ this.nonPrivilegedCluster = ClusterFactory.create(clusterConfiguration, CassandraConsistenciesConfiguration.DEFAULT);
KeyspaceConfiguration keyspaceConfiguration = KeyspaceConfiguration.builder()
.keyspace(KEYSPACE)
.replicationFactor(1)
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java
index e11113d..2ef8b92 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/DockerCassandra.java
@@ -21,6 +21,7 @@ package org.apache.james.backends.cassandra;
import org.apache.james.backends.cassandra.init.ClusterFactory;
import org.apache.james.backends.cassandra.init.KeyspaceFactory;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import org.apache.james.backends.cassandra.init.configuration.KeyspaceConfiguration;
import org.apache.james.util.Host;
@@ -34,7 +35,6 @@ import org.testcontainers.images.builder.dockerfile.DockerfileBuilder;
import com.datastax.driver.core.Cluster;
import com.datastax.driver.core.Session;
-import com.datastax.driver.core.schemabuilder.SchemaBuilder;
import com.github.dockerjava.api.DockerClient;
import com.google.common.collect.ImmutableMap;
@@ -59,28 +59,14 @@ public class DockerCassandra {
}
public void initializeKeyspace(KeyspaceConfiguration configuration) {
- try (Cluster privilegedCluster = ClusterFactory.create(cassandra.superUserConfigurationBuilder().build())) {
+ try (Cluster privilegedCluster = ClusterFactory.create(cassandra.superUserConfigurationBuilder().build(),
+ CassandraConsistenciesConfiguration.DEFAULT)) {
provisionNonPrivilegedUser(privilegedCluster);
KeyspaceFactory.createKeyspace(configuration, privilegedCluster);
grantPermissionToTestingUser(privilegedCluster, configuration.getKeyspace());
}
}
- public void dropKeyspace(String keyspace) {
- try (Cluster cluster = ClusterFactory.create(cassandra.superUserConfigurationBuilder().build())) {
- try (Session cassandraSession = cluster.newSession()) {
- boolean applied = cassandraSession.execute(
- SchemaBuilder.dropKeyspace(keyspace)
- .ifExists())
- .wasApplied();
-
- if (!applied) {
- throw new IllegalStateException("cannot drop keyspace '" + keyspace + "'");
- }
- }
- }
- }
-
private void provisionNonPrivilegedUser(Cluster privilegedCluster) {
try (Session session = privilegedCluster.newSession()) {
session.execute("CREATE ROLE IF NOT EXISTS " + CASSANDRA_TESTING_USER + " WITH PASSWORD = '" + CASSANDRA_TESTING_PASSWORD + "' AND LOGIN = true");
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java
index 7799644..11e30cd 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ClusterFactoryTest.java
@@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.backends.cassandra.DockerCassandra;
import org.apache.james.backends.cassandra.components.CassandraModule;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -47,7 +48,7 @@ class ClusterFactoryTest {
@Test
void consistencyLevelShouldBeEqualToQuorum(DockerCassandra dockerCassandra) {
Cluster cluster = ClusterFactory.create(dockerCassandra.configurationBuilder()
- .build());
+ .build(), CassandraConsistenciesConfiguration.DEFAULT);
ConsistencyLevel consistencyLevel = cluster.getConfiguration()
.getQueryOptions()
@@ -62,14 +63,14 @@ class ClusterFactoryTest {
assertThatThrownBy(() -> ClusterFactory.create(
dockerCassandra.configurationBuilder()
- .build()))
+ .build(), CassandraConsistenciesConfiguration.DEFAULT))
.isInstanceOf(NoHostAvailableException.class);
}
@Test
void createShouldReturnAContactableCluster(DockerCassandra dockerCassandra) {
Cluster cluster = ClusterFactory.create(dockerCassandra.configurationBuilder()
- .build());
+ .build(), CassandraConsistenciesConfiguration.DEFAULT);
assertThatClusterIsContactable(cluster);
}
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ResilientClusterProviderTest.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ResilientClusterProviderTest.java
index 29142d0..13dd5f7 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ResilientClusterProviderTest.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/ResilientClusterProviderTest.java
@@ -26,6 +26,7 @@ import java.time.Duration;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.backends.cassandra.components.CassandraModule;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
@@ -38,7 +39,7 @@ class ResilientClusterProviderTest {
@Test
void getShouldNotThrowWhenHealthyCassandra() {
- assertThatCode(() -> new ResilientClusterProvider(cassandraExtension.clusterConfiguration().build())
+ assertThatCode(() -> new ResilientClusterProvider(cassandraExtension.clusterConfiguration().build(), CassandraConsistenciesConfiguration.DEFAULT)
.get())
.doesNotThrowAnyException();
}
@@ -50,7 +51,7 @@ class ResilientClusterProviderTest {
assertThatThrownBy(() -> new ResilientClusterProvider(cassandraExtension.clusterConfiguration()
.maxRetry(1)
.minDelay(1)
- .build())
+ .build(), CassandraConsistenciesConfiguration.DEFAULT)
.get())
.isInstanceOf(Exception.class);
} finally {
@@ -68,7 +69,8 @@ class ResilientClusterProviderTest {
.subscribeOn(Schedulers.elastic())
.subscribe();
- assertThatCode(() -> new ResilientClusterProvider(cassandraExtension.clusterConfiguration().build())
+ assertThatCode(() -> new ResilientClusterProvider(cassandraExtension.clusterConfiguration().build(),
+ CassandraConsistenciesConfiguration.DEFAULT)
.get())
.doesNotThrowAnyException();
} finally {
diff --git a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/SessionWithInitializedTablesFactoryTest.java b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/SessionWithInitializedTablesFactoryTest.java
index 84a1a71..8f2ae89 100644
--- a/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/SessionWithInitializedTablesFactoryTest.java
+++ b/backends-common/cassandra/src/test/java/org/apache/james/backends/cassandra/init/SessionWithInitializedTablesFactoryTest.java
@@ -29,6 +29,7 @@ import java.util.function.Supplier;
import org.apache.james.backends.cassandra.DockerCassandra;
import org.apache.james.backends.cassandra.DockerCassandraExtension;
import org.apache.james.backends.cassandra.components.CassandraModule;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import org.apache.james.backends.cassandra.init.configuration.KeyspaceConfiguration;
import org.apache.james.backends.cassandra.versions.CassandraSchemaVersionDAO;
@@ -123,7 +124,7 @@ class SessionWithInitializedTablesFactoryTest {
private static Supplier<Session> createSession(DockerCassandraExtension.DockerCassandra cassandraServer) {
ClusterConfiguration clusterConfiguration = DockerCassandra.configurationBuilder(cassandraServer.getHost())
.build();
- Cluster cluster = ClusterFactory.create(clusterConfiguration);
+ Cluster cluster = ClusterFactory.create(clusterConfiguration, CassandraConsistenciesConfiguration.DEFAULT);
KeyspaceConfiguration keyspaceConfiguration = DockerCassandra.mainKeyspaceConfiguration();
KeyspaceFactory.createKeyspace(keyspaceConfiguration, cluster);
return () -> new SessionWithInitializedTablesFactory(
diff --git a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/FixingGhostMailboxTest.java b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/FixingGhostMailboxTest.java
index bf73116..cd59cd5 100644
--- a/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/FixingGhostMailboxTest.java
+++ b/server/protocols/webadmin-integration-test/distributed-webadmin-integration-test/src/test/java/org/apache/james/webadmin/integration/rabbitmq/FixingGhostMailboxTest.java
@@ -51,6 +51,7 @@ import org.apache.james.JamesServerBuilder;
import org.apache.james.JamesServerExtension;
import org.apache.james.SearchConfiguration;
import org.apache.james.backends.cassandra.init.ClusterFactory;
+import org.apache.james.backends.cassandra.init.configuration.CassandraConsistenciesConfiguration;
import org.apache.james.backends.cassandra.init.configuration.ClusterConfiguration;
import org.apache.james.core.Username;
import org.apache.james.jmap.AccessToken;
@@ -161,7 +162,7 @@ class FixingGhostMailboxTest {
CassandraProbe probe = server.getProbe(CassandraProbe.class);
ClusterConfiguration cassandraConfiguration = probe.getConfiguration();
- try (Cluster cluster = ClusterFactory.create(cassandraConfiguration)) {
+ try (Cluster cluster = ClusterFactory.create(cassandraConfiguration, CassandraConsistenciesConfiguration.DEFAULT)) {
try (Session session = cluster.connect(probe.getMainKeyspaceConfiguration().getKeyspace())) {
simulateGhostMailboxBug(session);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 02/07: JAMES-3314 implement Passtrough blobstore
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 5e2756371c91e93bced13b795af4cb9a1403681a
Author: Rémi Kowalski <rk...@linagora.com>
AuthorDate: Tue Jul 21 13:48:03 2020 +0200
JAMES-3314 implement Passtrough blobstore
---
.../james/vault/DeletedMessageVaultHookTest.java | 5 +-
.../blob/BlobStoreDeletedMessageVaultTest.java | 5 +-
.../blob/cassandra/CassandraBlobStoreFactory.java | 17 ++---
.../blob/cassandra/CassandraBlobStoreTest.java | 10 +--
....java => CassandraPassTroughBlobStoreTest.java} | 13 ++--
.../file/LocalFileBlobExportMechanismTest.java | 5 +-
.../james/blob/memory/MemoryBlobStoreFactory.java | 15 +----
.../memory/MemoryBlobStorePassThroughTest.java | 7 ++-
.../james/blob/memory/MemoryBlobStoreTest.java | 5 +-
.../blob/deduplication/BlobStoreFactory.java | 72 ++++++++++++++++++++++
.../blob/deduplication/StorageStrategy.java} | 21 ++-----
.../blob/deduplication/PassThroughBlobStore.scala | 38 +++---------
.../james/blob/mail/MimeMessageStoreTest.java | 5 +-
.../routes/DeletedMessagesVaultRoutesTest.java | 5 +-
.../james/webadmin/service/ExportServiceTest.java | 5 --
.../webadmin/service/ExportServiceTestSystem.java | 5 +-
.../linshare/LinshareBlobExportMechanismTest.java | 5 +-
17 files changed, 143 insertions(+), 95 deletions(-)
diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java
index 766fdd9..0fa94af 100644
--- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java
+++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/DeletedMessageVaultHookTest.java
@@ -109,7 +109,10 @@ class DeletedMessageVaultHookTest {
void setUp() throws Exception {
clock = Clock.fixed(DELETION_DATE.toInstant(), ZoneOffset.UTC);
messageVault = new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(),
- MemoryBlobStoreFactory.create(new HashBlobId.Factory()), new BucketNameGenerator(clock), clock,
+ MemoryBlobStoreFactory.builder()
+ .blobIdFactory(new HashBlobId.Factory())
+ .defaultBucketName()
+ .passthrough(), new BucketNameGenerator(clock), clock,
RetentionConfiguration.DEFAULT);
DeletedMessageConverter deletedMessageConverter = new DeletedMessageConverter();
diff --git a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java
index a965a89..28b6506 100644
--- a/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java
+++ b/mailbox/plugin/deleted-messages-vault/src/test/java/org/apache/james/vault/blob/BlobStoreDeletedMessageVaultTest.java
@@ -64,7 +64,10 @@ class BlobStoreDeletedMessageVaultTest implements DeletedMessageVaultContract, D
clock = new UpdatableTickingClock(NOW.toInstant());
metricFactory = new RecordingMetricFactory();
messageVault = new BlobStoreDeletedMessageVault(metricFactory, new MemoryDeletedMessageMetadataVault(),
- MemoryBlobStoreFactory.create(new HashBlobId.Factory()),
+ MemoryBlobStoreFactory.builder()
+ .blobIdFactory(new HashBlobId.Factory())
+ .defaultBucketName()
+ .passthrough(),
new BucketNameGenerator(clock), clock, RetentionConfiguration.DEFAULT);
}
diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobStoreFactory.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobStoreFactory.java
index cc51047..f1d0576 100644
--- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobStoreFactory.java
+++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/CassandraBlobStoreFactory.java
@@ -23,24 +23,19 @@ import org.apache.james.backends.cassandra.init.configuration.CassandraConfigura
import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
-import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import com.datastax.driver.core.Session;
public class CassandraBlobStoreFactory {
- public static BlobStore forTesting(HashBlobId.Factory blobIdFactory,
- BucketName defaultBucketName,
- CassandraDumbBlobStore dumbBlobStore) {
- return new DeDuplicationBlobStore(dumbBlobStore, defaultBucketName, blobIdFactory);
- }
-
public static BlobStore forTesting(Session session) {
HashBlobId.Factory blobIdFactory = new HashBlobId.Factory();
CassandraBucketDAO bucketDAO = new CassandraBucketDAO(blobIdFactory, session);
CassandraDefaultBucketDAO defaultBucketDAO = new CassandraDefaultBucketDAO(session);
- return forTesting(
- blobIdFactory,
- BucketName.DEFAULT,
- new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, CassandraConfiguration.DEFAULT_CONFIGURATION, BucketName.DEFAULT));
+ CassandraDumbBlobStore dumbBlobStore = new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, CassandraConfiguration.DEFAULT_CONFIGURATION, BucketName.DEFAULT);
+ return BlobStoreFactory.builder().dumbBlobStore(dumbBlobStore)
+ .blobIdFactory(blobIdFactory)
+ .defaultBucketName()
+ .passthrough();
}
}
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
index cf43ad5..ae9136b 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
@@ -40,6 +40,7 @@ import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
import org.apache.james.blob.api.ObjectStoreException;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.apache.james.util.io.ZeroedInputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -71,10 +72,11 @@ public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
.build();
testee = new MetricableBlobStore(
metricsTestExtension.getMetricFactory(),
- CassandraBlobStoreFactory.forTesting(
- blobIdFactory,
- BucketName.DEFAULT,
- new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, cassandraConfiguration, BucketName.DEFAULT)));
+ BlobStoreFactory.builder()
+ .dumbBlobStore(new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, cassandraConfiguration, BucketName.DEFAULT))
+ .blobIdFactory(blobIdFactory)
+ .defaultBucketName()
+ .deduplication());
}
@Override
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java
similarity index 93%
copy from server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
copy to server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java
index cf43ad5..9ea1731 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java
@@ -40,6 +40,7 @@ import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
import org.apache.james.blob.api.ObjectStoreException;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.apache.james.util.io.ZeroedInputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -48,10 +49,9 @@ import org.junit.jupiter.api.extension.RegisterExtension;
import com.google.common.base.Strings;
import com.google.common.hash.Hashing;
import com.google.common.hash.HashingInputStream;
-
import reactor.core.publisher.Mono;
-public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
+public class CassandraPassTroughBlobStoreTest implements MetricableBlobStoreContract {
private static final int CHUNK_SIZE = 10240;
private static final int MULTIPLE_CHUNK_SIZE = 3;
@@ -71,10 +71,11 @@ public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
.build();
testee = new MetricableBlobStore(
metricsTestExtension.getMetricFactory(),
- CassandraBlobStoreFactory.forTesting(
- blobIdFactory,
- BucketName.DEFAULT,
- new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, cassandraConfiguration, BucketName.DEFAULT)));
+ BlobStoreFactory.builder()
+ .dumbBlobStore(new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, cassandraConfiguration, BucketName.DEFAULT))
+ .blobIdFactory(blobIdFactory)
+ .defaultBucketName()
+ .passthrough());
}
@Override
diff --git a/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java b/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java
index 4eff0c4..5655d0e 100644
--- a/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java
+++ b/server/blob/blob-export-file/src/test/java/org/apache/james/blob/export/file/LocalFileBlobExportMechanismTest.java
@@ -67,7 +67,10 @@ class LocalFileBlobExportMechanismTest {
@BeforeEach
void setUp(FileSystem fileSystem) throws Exception {
mailetContext = FakeMailContext.builder().postmaster(MailAddressFixture.POSTMASTER_AT_JAMES).build();
- blobStore = MemoryBlobStoreFactory.create(new HashBlobId.Factory());
+ blobStore = MemoryBlobStoreFactory.builder()
+ .blobIdFactory(new HashBlobId.Factory())
+ .defaultBucketName()
+ .passthrough();
InetAddress localHost = mock(InetAddress.class);
when(localHost.getHostName()).thenReturn(JAMES_HOST);
diff --git a/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java b/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java
index 5e7c7e2..bc59922 100644
--- a/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java
+++ b/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java
@@ -19,19 +19,10 @@
package org.apache.james.blob.memory;
-import org.apache.james.blob.api.BlobId;
-import org.apache.james.blob.api.BlobStore;
-import org.apache.james.blob.api.BucketName;
-import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore;
+import org.apache.james.server.blob.deduplication.BlobStoreFactory;
public class MemoryBlobStoreFactory {
- public static BlobStore create(BlobId.Factory factory) {
- return create(factory, BucketName.DEFAULT);
- }
-
- public static BlobStore create(BlobId.Factory factory, BucketName defaultBucketName) {
- return new DeDuplicationBlobStore(
- new MemoryDumbBlobStore(),
- defaultBucketName, factory);
+ public static BlobStoreFactory.RequireBlobIdFactory builder() {
+ return BlobStoreFactory.builder().dumbBlobStore(new MemoryDumbBlobStore());
}
}
diff --git a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java
index 80dba7e..aa72550 100644
--- a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java
+++ b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStorePassThroughTest.java
@@ -26,14 +26,17 @@ import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
import org.junit.jupiter.api.BeforeEach;
-public class MemoryBlobStoreTest implements MetricableBlobStoreContract {
+public class MemoryBlobStorePassThroughTest implements MetricableBlobStoreContract {
private static final HashBlobId.Factory BLOB_ID_FACTORY = new HashBlobId.Factory();
private BlobStore blobStore;
@BeforeEach
void setUp() {
- blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.create(BLOB_ID_FACTORY));
+ blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.builder()
+ .blobIdFactory(BLOB_ID_FACTORY)
+ .defaultBucketName()
+ .passthrough());
}
@Override
diff --git a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
index 80dba7e..147133c 100644
--- a/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
+++ b/server/blob/blob-memory/src/test/java/org/apache/james/blob/memory/MemoryBlobStoreTest.java
@@ -33,7 +33,10 @@ public class MemoryBlobStoreTest implements MetricableBlobStoreContract {
@BeforeEach
void setUp() {
- blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.create(BLOB_ID_FACTORY));
+ blobStore = new MetricableBlobStore(metricsTestExtension.getMetricFactory(), new MetricableBlobStore(metricsTestExtension.getMetricFactory(), MemoryBlobStoreFactory.builder()
+ .blobIdFactory(BLOB_ID_FACTORY)
+ .defaultBucketName()
+ .deduplication()));
}
@Override
diff --git a/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobStoreFactory.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobStoreFactory.java
new file mode 100644
index 0000000..4a3fdc3
--- /dev/null
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/BlobStoreFactory.java
@@ -0,0 +1,72 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ***************************************************************/
+
+package org.apache.james.server.blob.deduplication;
+
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BlobStore;
+import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.DumbBlobStore;
+
+public abstract class BlobStoreFactory {
+ @FunctionalInterface
+ public interface RequireDumbBlobStore {
+ RequireBlobIdFactory dumbBlobStore(DumbBlobStore dumbBlobStore);
+ }
+
+ @FunctionalInterface
+ public interface RequireBlobIdFactory {
+ RequireBucketName blobIdFactory(BlobId.Factory blobIdFactory);
+ }
+
+ @FunctionalInterface
+ public interface RequireBucketName {
+ RequireStoringStrategy bucket(BucketName defaultBucketName);
+
+ default RequireStoringStrategy defaultBucketName() {
+ return bucket(BucketName.DEFAULT);
+ }
+ }
+
+ @FunctionalInterface
+ public interface RequireStoringStrategy {
+ BlobStore strategy(StorageStrategy storageStrategy);
+
+ default BlobStore passthrough() {
+ return strategy(StorageStrategy.PASSTHROUGH);
+ }
+
+ default BlobStore deduplication() {
+ return strategy(StorageStrategy.DEDUPLICATION);
+ }
+ }
+
+ public static RequireDumbBlobStore builder() {
+ return dumbBlobStore -> blobIdFactory -> defaultBucketName -> storageStrategy -> {
+ switch (storageStrategy) {
+ case PASSTHROUGH:
+ return new PassThroughBlobStore(dumbBlobStore, defaultBucketName, blobIdFactory);
+ case DEDUPLICATION:
+ return new DeDuplicationBlobStore(dumbBlobStore, defaultBucketName, blobIdFactory);
+ default:
+ throw new IllegalArgumentException("Unknown storage strategy");
+ }
+ };
+ }
+}
diff --git a/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/StorageStrategy.java
similarity index 65%
copy from server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java
copy to server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/StorageStrategy.java
index 5e7c7e2..91d2df1 100644
--- a/server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreFactory.java
+++ b/server/blob/blob-storage-strategy/src/main/java/org/apache/james/server/blob/deduplication/StorageStrategy.java
@@ -16,22 +16,9 @@
* specific language governing permissions and limitations *
* under the License. *
****************************************************************/
+package org.apache.james.server.blob.deduplication;
-package org.apache.james.blob.memory;
-
-import org.apache.james.blob.api.BlobId;
-import org.apache.james.blob.api.BlobStore;
-import org.apache.james.blob.api.BucketName;
-import org.apache.james.server.blob.deduplication.DeDuplicationBlobStore;
-
-public class MemoryBlobStoreFactory {
- public static BlobStore create(BlobId.Factory factory) {
- return create(factory, BucketName.DEFAULT);
- }
-
- public static BlobStore create(BlobId.Factory factory, BucketName defaultBucketName) {
- return new DeDuplicationBlobStore(
- new MemoryDumbBlobStore(),
- defaultBucketName, factory);
- }
+public enum StorageStrategy {
+ PASSTHROUGH,
+ DEDUPLICATION
}
diff --git a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
index 5bd663f..5f29ffd 100644
--- a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
+++ b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
@@ -22,31 +22,21 @@ package org.apache.james.server.blob.deduplication
import java.io.InputStream
import com.google.common.base.Preconditions
-import com.google.common.hash.{Hashing, HashingInputStream}
-import com.google.common.io.{ByteSource, FileBackedOutputStream}
import javax.inject.{Inject, Named}
-import org.apache.commons.io.IOUtils
import org.apache.james.blob.api.{BlobId, BlobStore, BucketName, DumbBlobStore}
import org.reactivestreams.Publisher
-import reactor.core.publisher.Mono
import reactor.core.scala.publisher.SMono
-import reactor.util.function.{Tuple2, Tuples}
-object DeDuplicationBlobStore {
- val DEFAULT_BUCKET = "defaultBucket"
- val LAZY_RESOURCE_CLEANUP = false
- val FILE_THRESHOLD = 10000
-}
-class DeDuplicationBlobStore @Inject()(dumbBlobStore: DumbBlobStore,
- @Named("defaultBucket") defaultBucketName: BucketName,
- blobIdFactory: BlobId.Factory) extends BlobStore {
+class PassThroughBlobStore @Inject()(dumbBlobStore: DumbBlobStore,
+ @Named("defaultBucket") defaultBucketName: BucketName,
+ blobIdFactory: BlobId.Factory) extends BlobStore {
override def save(bucketName: BucketName, data: Array[Byte], storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = {
Preconditions.checkNotNull(bucketName)
Preconditions.checkNotNull(data)
- val blobId = blobIdFactory.forPayload(data)
+ val blobId = blobIdFactory.randomId()
SMono(dumbBlobStore.save(bucketName, blobId, data))
.`then`(SMono.just(blobId))
@@ -55,23 +45,11 @@ class DeDuplicationBlobStore @Inject()(dumbBlobStore: DumbBlobStore,
override def save(bucketName: BucketName, data: InputStream, storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = {
Preconditions.checkNotNull(bucketName)
Preconditions.checkNotNull(data)
- val hashingInputStream = new HashingInputStream(Hashing.sha256, data)
- val sourceSupplier: FileBackedOutputStream => SMono[BlobId] = (fileBackedOutputStream: FileBackedOutputStream) => saveAndGenerateBlobId(bucketName, hashingInputStream, fileBackedOutputStream)
- Mono.using(() => new FileBackedOutputStream(DeDuplicationBlobStore.FILE_THRESHOLD),
- sourceSupplier,
- (fileBackedOutputStream: FileBackedOutputStream) => fileBackedOutputStream.reset(),
- DeDuplicationBlobStore.LAZY_RESOURCE_CLEANUP)
- }
-
- private def saveAndGenerateBlobId(bucketName: BucketName, hashingInputStream: HashingInputStream, fileBackedOutputStream: FileBackedOutputStream): SMono[BlobId] =
- SMono.fromCallable(() => {
- IOUtils.copy(hashingInputStream, fileBackedOutputStream)
- Tuples.of(blobIdFactory.from(hashingInputStream.hash.toString), fileBackedOutputStream.asByteSource)
- })
- .flatMap((tuple: Tuple2[BlobId, ByteSource]) =>
- SMono(dumbBlobStore.save(bucketName, tuple.getT1, tuple.getT2))
- .`then`(SMono.just(tuple.getT1)))
+ val blobId = blobIdFactory.randomId()
+ SMono(dumbBlobStore.save(bucketName, blobId, data))
+ .`then`(SMono.just(blobId))
+ }
override def readBytes(bucketName: BucketName, blobId: BlobId): Publisher[Array[Byte]] = {
Preconditions.checkNotNull(bucketName)
diff --git a/server/blob/mail-store/src/test/java/org/apache/james/blob/mail/MimeMessageStoreTest.java b/server/blob/mail-store/src/test/java/org/apache/james/blob/mail/MimeMessageStoreTest.java
index e1dda9a..bb24ab5 100644
--- a/server/blob/mail-store/src/test/java/org/apache/james/blob/mail/MimeMessageStoreTest.java
+++ b/server/blob/mail-store/src/test/java/org/apache/james/blob/mail/MimeMessageStoreTest.java
@@ -47,7 +47,10 @@ class MimeMessageStoreTest {
@BeforeEach
void setUp() {
- blobStore = MemoryBlobStoreFactory.create(BLOB_ID_FACTORY);
+ blobStore = MemoryBlobStoreFactory.builder()
+ .blobIdFactory(BLOB_ID_FACTORY)
+ .defaultBucketName()
+ .passthrough();
testee = MimeMessageStore.factory(blobStore).mimeMessageStore();
}
diff --git a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
index 43004e0..3cf3d00 100644
--- a/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox-deleted-message-vault/src/test/java/org/apache/james/webadmin/vault/routes/DeletedMessagesVaultRoutesTest.java
@@ -176,7 +176,10 @@ class DeletedMessagesVaultRoutesTest {
@BeforeEach
void beforeEach() throws Exception {
blobIdFactory = new HashBlobId.Factory();
- blobStore = spy(MemoryBlobStoreFactory.create(blobIdFactory));
+ blobStore = spy(MemoryBlobStoreFactory.builder()
+ .blobIdFactory(blobIdFactory)
+ .defaultBucketName()
+ .passthrough());
clock = new UpdatableTickingClock(OLD_DELETION_DATE.toInstant());
vault = spy(new BlobStoreDeletedMessageVault(new RecordingMetricFactory(), new MemoryDeletedMessageMetadataVault(),
blobStore, new BucketNameGenerator(clock), clock,
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java
index fa0c391..c016b59 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTest.java
@@ -206,11 +206,6 @@ class ExportServiceTest {
Task.Result result = testee.export(progress, BOB).block();
- String fileName = Files.getNameWithoutExtension(getFileUrl());
- String blobId = fileName.substring(fileName.lastIndexOf("-") + 1);
-
- testSystem.blobStore.read(testSystem.blobStore.getDefaultBucketName(), FACTORY.from(blobId));
-
assertThat(result).isEqualTo(Task.Result.COMPLETED);
}
diff --git a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTestSystem.java b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTestSystem.java
index 389ffad..c7b0b94 100644
--- a/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTestSystem.java
+++ b/server/protocols/webadmin/webadmin-mailbox/src/test/java/org/apache/james/webadmin/service/ExportServiceTestSystem.java
@@ -76,7 +76,10 @@ public class ExportServiceTestSystem {
bobSession = mailboxManager.createSystemSession(BOB);
- blobStore = Mockito.spy(MemoryBlobStoreFactory.create(FACTORY));
+ blobStore = Mockito.spy(MemoryBlobStoreFactory.builder()
+ .blobIdFactory(FACTORY)
+ .defaultBucketName()
+ .passthrough());
mailetContext = FakeMailContext.builder().postmaster(MailAddressFixture.POSTMASTER_AT_JAMES).build();
blobExport = new LocalFileBlobExportMechanism(mailetContext, blobStore, fileSystem, dnsService,
LocalFileBlobExportMechanism.Configuration.DEFAULT_CONFIGURATION);
diff --git a/third-party/linshare/src/test/java/org/apache/james/linshare/LinshareBlobExportMechanismTest.java b/third-party/linshare/src/test/java/org/apache/james/linshare/LinshareBlobExportMechanismTest.java
index 738d4b0..4ed698e 100644
--- a/third-party/linshare/src/test/java/org/apache/james/linshare/LinshareBlobExportMechanismTest.java
+++ b/third-party/linshare/src/test/java/org/apache/james/linshare/LinshareBlobExportMechanismTest.java
@@ -58,7 +58,10 @@ class LinshareBlobExportMechanismTest {
@BeforeEach
void setUp() throws Exception {
blobIdFactory = new HashBlobId.Factory();
- blobStore = MemoryBlobStoreFactory.create(blobIdFactory);
+ blobStore = MemoryBlobStoreFactory.builder()
+ .blobIdFactory(blobIdFactory)
+ .defaultBucketName()
+ .passthrough();
testee = new LinshareBlobExportMechanism(
linshareExtension.getDelegationAccountAPI(),
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org
[james-project] 03/07: JAMES-3314 Mutualize
CassandraBlobStoreContract
Posted by bt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 700116f79c8cc5f71279210142161719c0302152
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Wed Jul 22 11:48:49 2020 +0700
JAMES-3314 Mutualize CassandraBlobStoreContract
---
...reTest.java => CassandraBlobStoreContract.java} | 84 ++++++---------------
.../blob/cassandra/CassandraBlobStoreTest.java | 88 +---------------------
.../CassandraPassTroughBlobStoreTest.java | 87 +--------------------
3 files changed, 29 insertions(+), 230 deletions(-)
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreContract.java
similarity index 50%
copy from server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
copy to server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreContract.java
index ae9136b..b6a6434 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreContract.java
@@ -22,7 +22,6 @@ package org.apache.james.blob.cassandra;
import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
import java.io.IOException;
@@ -30,21 +29,11 @@ import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import org.apache.commons.io.IOUtils;
-import org.apache.james.backends.cassandra.CassandraCluster;
-import org.apache.james.backends.cassandra.CassandraClusterExtension;
-import org.apache.james.backends.cassandra.init.configuration.CassandraConfiguration;
import org.apache.james.blob.api.BlobId;
-import org.apache.james.blob.api.BlobStore;
-import org.apache.james.blob.api.BucketName;
-import org.apache.james.blob.api.HashBlobId;
-import org.apache.james.blob.api.MetricableBlobStore;
import org.apache.james.blob.api.MetricableBlobStoreContract;
import org.apache.james.blob.api.ObjectStoreException;
-import org.apache.james.server.blob.deduplication.BlobStoreFactory;
import org.apache.james.util.io.ZeroedInputStream;
-import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.RegisterExtension;
import com.google.common.base.Strings;
import com.google.common.hash.Hashing;
@@ -52,93 +41,62 @@ import com.google.common.hash.HashingInputStream;
import reactor.core.publisher.Mono;
-public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
- private static final int CHUNK_SIZE = 10240;
- private static final int MULTIPLE_CHUNK_SIZE = 3;
-
- @RegisterExtension
- static CassandraClusterExtension cassandraCluster = new CassandraClusterExtension(CassandraBlobModule.MODULE);
-
- private BlobStore testee;
- private CassandraDefaultBucketDAO defaultBucketDAO;
-
- @BeforeEach
- void setUp(CassandraCluster cassandra) {
- HashBlobId.Factory blobIdFactory = new HashBlobId.Factory();
- CassandraBucketDAO bucketDAO = new CassandraBucketDAO(blobIdFactory, cassandra.getConf());
- defaultBucketDAO = spy(new CassandraDefaultBucketDAO(cassandra.getConf()));
- CassandraConfiguration cassandraConfiguration = CassandraConfiguration.builder()
- .blobPartSize(CHUNK_SIZE)
- .build();
- testee = new MetricableBlobStore(
- metricsTestExtension.getMetricFactory(),
- BlobStoreFactory.builder()
- .dumbBlobStore(new CassandraDumbBlobStore(defaultBucketDAO, bucketDAO, cassandraConfiguration, BucketName.DEFAULT))
- .blobIdFactory(blobIdFactory)
- .defaultBucketName()
- .deduplication());
- }
-
- @Override
- public BlobStore testee() {
- return testee;
- }
+public interface CassandraBlobStoreContract extends MetricableBlobStoreContract {
+ int MULTIPLE_CHUNK_SIZE = 3;
+ int CHUNK_SIZE = 10240;
- @Override
- public BlobId.Factory blobIdFactory() {
- return new HashBlobId.Factory();
- }
+ CassandraDefaultBucketDAO defaultBucketDAO();
@Test
- void readBytesShouldReturnSplitSavedDataByChunk() {
+ default void readBytesShouldReturnSplitSavedDataByChunk() {
String longString = Strings.repeat("0123456789\n", MULTIPLE_CHUNK_SIZE);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
+ BlobId blobId = Mono.from(testee().save(testee().getDefaultBucketName(), longString, LOW_COST)).block();
- byte[] bytes = Mono.from(testee.readBytes(testee.getDefaultBucketName(), blobId)).block();
+ byte[] bytes = Mono.from(testee().readBytes(testee().getDefaultBucketName(), blobId)).block();
assertThat(new String(bytes, StandardCharsets.UTF_8)).isEqualTo(longString);
}
@Test
- void readBytesShouldNotReturnInvalidResultsWhenPartialDataPresent() {
+ default void readBytesShouldNotReturnInvalidResultsWhenPartialDataPresent() {
int repeatCount = MULTIPLE_CHUNK_SIZE * CHUNK_SIZE;
String longString = Strings.repeat("0123456789\n", repeatCount);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
+ BlobId blobId = Mono.from(testee().save(testee().getDefaultBucketName(), longString, LOW_COST)).block();
- when(defaultBucketDAO.readPart(blobId, 1)).thenReturn(Mono.empty());
+ when(defaultBucketDAO().readPart(blobId, 1)).thenReturn(Mono.empty());
- assertThatThrownBy(() -> Mono.from(testee.readBytes(testee.getDefaultBucketName(), blobId)).block())
+ assertThatThrownBy(() -> Mono.from(testee().readBytes(testee().getDefaultBucketName(), blobId)).block())
.isInstanceOf(ObjectStoreException.class)
.hasMessageContaining("Missing blob part for blobId");
}
@Test
- void readShouldNotReturnInvalidResultsWhenPartialDataPresent() {
+ default void readShouldNotReturnInvalidResultsWhenPartialDataPresent() {
int repeatCount = MULTIPLE_CHUNK_SIZE * CHUNK_SIZE;
String longString = Strings.repeat("0123456789\n", repeatCount);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
+ BlobId blobId = Mono.from(testee().save(testee().getDefaultBucketName(), longString, LOW_COST)).block();
- when(defaultBucketDAO.readPart(blobId, 1)).thenReturn(Mono.empty());
+ when(defaultBucketDAO().readPart(blobId, 1)).thenReturn(Mono.empty());
- assertThatThrownBy(() -> IOUtils.toString(testee.read(testee.getDefaultBucketName(), blobId), StandardCharsets.UTF_8))
+ assertThatThrownBy(() -> IOUtils.toString(testee().read(testee().getDefaultBucketName(), blobId), StandardCharsets.UTF_8))
.isInstanceOf(ObjectStoreException.class)
.hasMessageContaining("Missing blob part for blobId");
}
@Test
- void deleteBucketShouldThrowWhenDeletingDefaultBucket() {
- assertThatThrownBy(() -> testee.deleteBucket(testee.getDefaultBucketName()))
+ default void deleteBucketShouldThrowWhenDeletingDefaultBucket() {
+ assertThatThrownBy(() -> testee().deleteBucket(testee().getDefaultBucketName()))
.isInstanceOf(IllegalArgumentException.class)
.hasMessage("Deleting the default bucket is forbidden");
}
@Test
- void blobStoreShouldSupport100MBBlob() throws IOException {
+ default void blobStoreShouldSupport100MBBlob() throws IOException {
ZeroedInputStream data = new ZeroedInputStream(100_000_000);
HashingInputStream writeHash = new HashingInputStream(Hashing.sha256(), data);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), writeHash, LOW_COST)).block();
+ BlobId blobId = Mono.from(testee().save(testee().getDefaultBucketName(), writeHash, LOW_COST)).block();
- InputStream bytes = testee.read(testee.getDefaultBucketName(), blobId);
+ InputStream bytes = testee().read(testee().getDefaultBucketName(), blobId);
HashingInputStream readHash = new HashingInputStream(Hashing.sha256(), bytes);
consumeStream(readHash);
@@ -151,4 +109,4 @@ public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
// consume the rest of the stream
}
}
-}
\ No newline at end of file
+}
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
index ae9136b..73b031a 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraBlobStoreTest.java
@@ -19,17 +19,8 @@
package org.apache.james.blob.cassandra;
-import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.commons.io.IOUtils;
import org.apache.james.backends.cassandra.CassandraCluster;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.backends.cassandra.init.configuration.CassandraConfiguration;
@@ -38,24 +29,11 @@ import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
-import org.apache.james.blob.api.MetricableBlobStoreContract;
-import org.apache.james.blob.api.ObjectStoreException;
import org.apache.james.server.blob.deduplication.BlobStoreFactory;
-import org.apache.james.util.io.ZeroedInputStream;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
-import com.google.common.base.Strings;
-import com.google.common.hash.Hashing;
-import com.google.common.hash.HashingInputStream;
-
-import reactor.core.publisher.Mono;
-
-public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
- private static final int CHUNK_SIZE = 10240;
- private static final int MULTIPLE_CHUNK_SIZE = 3;
-
+public class CassandraBlobStoreTest implements CassandraBlobStoreContract {
@RegisterExtension
static CassandraClusterExtension cassandraCluster = new CassandraClusterExtension(CassandraBlobModule.MODULE);
@@ -89,66 +67,8 @@ public class CassandraBlobStoreTest implements MetricableBlobStoreContract {
return new HashBlobId.Factory();
}
- @Test
- void readBytesShouldReturnSplitSavedDataByChunk() {
- String longString = Strings.repeat("0123456789\n", MULTIPLE_CHUNK_SIZE);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
-
- byte[] bytes = Mono.from(testee.readBytes(testee.getDefaultBucketName(), blobId)).block();
-
- assertThat(new String(bytes, StandardCharsets.UTF_8)).isEqualTo(longString);
- }
-
- @Test
- void readBytesShouldNotReturnInvalidResultsWhenPartialDataPresent() {
- int repeatCount = MULTIPLE_CHUNK_SIZE * CHUNK_SIZE;
- String longString = Strings.repeat("0123456789\n", repeatCount);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
-
- when(defaultBucketDAO.readPart(blobId, 1)).thenReturn(Mono.empty());
-
- assertThatThrownBy(() -> Mono.from(testee.readBytes(testee.getDefaultBucketName(), blobId)).block())
- .isInstanceOf(ObjectStoreException.class)
- .hasMessageContaining("Missing blob part for blobId");
- }
-
- @Test
- void readShouldNotReturnInvalidResultsWhenPartialDataPresent() {
- int repeatCount = MULTIPLE_CHUNK_SIZE * CHUNK_SIZE;
- String longString = Strings.repeat("0123456789\n", repeatCount);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
-
- when(defaultBucketDAO.readPart(blobId, 1)).thenReturn(Mono.empty());
-
- assertThatThrownBy(() -> IOUtils.toString(testee.read(testee.getDefaultBucketName(), blobId), StandardCharsets.UTF_8))
- .isInstanceOf(ObjectStoreException.class)
- .hasMessageContaining("Missing blob part for blobId");
- }
-
- @Test
- void deleteBucketShouldThrowWhenDeletingDefaultBucket() {
- assertThatThrownBy(() -> testee.deleteBucket(testee.getDefaultBucketName()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Deleting the default bucket is forbidden");
- }
-
- @Test
- void blobStoreShouldSupport100MBBlob() throws IOException {
- ZeroedInputStream data = new ZeroedInputStream(100_000_000);
- HashingInputStream writeHash = new HashingInputStream(Hashing.sha256(), data);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), writeHash, LOW_COST)).block();
-
- InputStream bytes = testee.read(testee.getDefaultBucketName(), blobId);
- HashingInputStream readHash = new HashingInputStream(Hashing.sha256(), bytes);
- consumeStream(readHash);
-
- assertThat(readHash.hash().toString()).isEqualTo(writeHash.hash().toString());
- }
-
- private void consumeStream(InputStream tmpMsgIn) throws IOException {
- byte[] discard = new byte[4096];
- while (tmpMsgIn.read(discard) != -1) {
- // consume the rest of the stream
- }
+ @Override
+ public CassandraDefaultBucketDAO defaultBucketDAO() {
+ return defaultBucketDAO;
}
}
\ No newline at end of file
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java
index 9ea1731..755dc28 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/CassandraPassTroughBlobStoreTest.java
@@ -19,17 +19,8 @@
package org.apache.james.blob.cassandra;
-import static org.apache.james.blob.api.BlobStore.StoragePolicy.LOW_COST;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
-
-import org.apache.commons.io.IOUtils;
import org.apache.james.backends.cassandra.CassandraCluster;
import org.apache.james.backends.cassandra.CassandraClusterExtension;
import org.apache.james.backends.cassandra.init.configuration.CassandraConfiguration;
@@ -38,23 +29,11 @@ import org.apache.james.blob.api.BlobStore;
import org.apache.james.blob.api.BucketName;
import org.apache.james.blob.api.HashBlobId;
import org.apache.james.blob.api.MetricableBlobStore;
-import org.apache.james.blob.api.MetricableBlobStoreContract;
-import org.apache.james.blob.api.ObjectStoreException;
import org.apache.james.server.blob.deduplication.BlobStoreFactory;
-import org.apache.james.util.io.ZeroedInputStream;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
-import com.google.common.base.Strings;
-import com.google.common.hash.Hashing;
-import com.google.common.hash.HashingInputStream;
-import reactor.core.publisher.Mono;
-
-public class CassandraPassTroughBlobStoreTest implements MetricableBlobStoreContract {
- private static final int CHUNK_SIZE = 10240;
- private static final int MULTIPLE_CHUNK_SIZE = 3;
-
+public class CassandraPassTroughBlobStoreTest implements CassandraBlobStoreContract {
@RegisterExtension
static CassandraClusterExtension cassandraCluster = new CassandraClusterExtension(CassandraBlobModule.MODULE);
@@ -88,66 +67,8 @@ public class CassandraPassTroughBlobStoreTest implements MetricableBlobStoreCont
return new HashBlobId.Factory();
}
- @Test
- void readBytesShouldReturnSplitSavedDataByChunk() {
- String longString = Strings.repeat("0123456789\n", MULTIPLE_CHUNK_SIZE);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
-
- byte[] bytes = Mono.from(testee.readBytes(testee.getDefaultBucketName(), blobId)).block();
-
- assertThat(new String(bytes, StandardCharsets.UTF_8)).isEqualTo(longString);
- }
-
- @Test
- void readBytesShouldNotReturnInvalidResultsWhenPartialDataPresent() {
- int repeatCount = MULTIPLE_CHUNK_SIZE * CHUNK_SIZE;
- String longString = Strings.repeat("0123456789\n", repeatCount);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
-
- when(defaultBucketDAO.readPart(blobId, 1)).thenReturn(Mono.empty());
-
- assertThatThrownBy(() -> Mono.from(testee.readBytes(testee.getDefaultBucketName(), blobId)).block())
- .isInstanceOf(ObjectStoreException.class)
- .hasMessageContaining("Missing blob part for blobId");
- }
-
- @Test
- void readShouldNotReturnInvalidResultsWhenPartialDataPresent() {
- int repeatCount = MULTIPLE_CHUNK_SIZE * CHUNK_SIZE;
- String longString = Strings.repeat("0123456789\n", repeatCount);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), longString, LOW_COST)).block();
-
- when(defaultBucketDAO.readPart(blobId, 1)).thenReturn(Mono.empty());
-
- assertThatThrownBy(() -> IOUtils.toString(testee.read(testee.getDefaultBucketName(), blobId), StandardCharsets.UTF_8))
- .isInstanceOf(ObjectStoreException.class)
- .hasMessageContaining("Missing blob part for blobId");
- }
-
- @Test
- void deleteBucketShouldThrowWhenDeletingDefaultBucket() {
- assertThatThrownBy(() -> testee.deleteBucket(testee.getDefaultBucketName()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Deleting the default bucket is forbidden");
- }
-
- @Test
- void blobStoreShouldSupport100MBBlob() throws IOException {
- ZeroedInputStream data = new ZeroedInputStream(100_000_000);
- HashingInputStream writeHash = new HashingInputStream(Hashing.sha256(), data);
- BlobId blobId = Mono.from(testee.save(testee.getDefaultBucketName(), writeHash, LOW_COST)).block();
-
- InputStream bytes = testee.read(testee.getDefaultBucketName(), blobId);
- HashingInputStream readHash = new HashingInputStream(Hashing.sha256(), bytes);
- consumeStream(readHash);
-
- assertThat(readHash.hash().toString()).isEqualTo(writeHash.hash().toString());
- }
-
- private void consumeStream(InputStream tmpMsgIn) throws IOException {
- byte[] discard = new byte[4096];
- while (tmpMsgIn.read(discard) != -1) {
- // consume the rest of the stream
- }
+ @Override
+ public CassandraDefaultBucketDAO defaultBucketDAO() {
+ return defaultBucketDAO;
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: server-dev-unsubscribe@james.apache.org
For additional commands, e-mail: server-dev-help@james.apache.org