You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by bt...@apache.org on 2021/03/08 05:44:12 UTC
[james-project] 02/30: JAMES-3506 Support saving byte source
This is an automated email from the ASF dual-hosted git repository.
btellier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit f3b5d25ad8b7eafa263cac1a213ee80864912d30
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Thu Feb 25 23:07:49 2021 +0700
JAMES-3506 Support saving byte source
---
.../java/org/apache/james/blob/api/BlobId.java | 4 ++
.../java/org/apache/james/blob/api/BlobStore.java | 4 ++
.../java/org/apache/james/blob/api/HashBlobId.java | 12 +++++
.../apache/james/blob/api/MetricableBlobStore.java | 7 +++
.../apache/james/blob/api/BlobStoreContract.java | 63 ++++++++++++++++++++++
.../org/apache/james/blob/api/HashBlobIdTest.java | 2 +-
.../java/org/apache/james/blob/api/TestBlobId.java | 7 +++
.../blob/cassandra/cache/CachedBlobStore.java | 23 ++++++++
.../blob/cassandra/cache/CachedBlobStoreTest.java | 47 ++++++++++++++++
.../deduplication/DeDuplicationBlobStore.scala | 12 ++++-
.../blob/deduplication/PassThroughBlobStore.scala | 10 ++++
11 files changed, 189 insertions(+), 2 deletions(-)
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java
index a6f6229..6996706 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobId.java
@@ -21,11 +21,15 @@ package org.apache.james.blob.api;
import java.util.UUID;
+import com.google.common.io.ByteSource;
+
public interface BlobId {
interface Factory {
BlobId forPayload(byte[] payload);
+ BlobId forPayload(ByteSource payload);
+
BlobId from(String id);
default BlobId randomId() {
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java
index faf45ad..e65fc33 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/BlobStore.java
@@ -23,6 +23,8 @@ import java.nio.charset.StandardCharsets;
import org.reactivestreams.Publisher;
+import com.google.common.io.ByteSource;
+
public interface BlobStore {
enum StoragePolicy {
@@ -35,6 +37,8 @@ public interface BlobStore {
Publisher<BlobId> save(BucketName bucketName, InputStream data, StoragePolicy storagePolicy);
+ Publisher<BlobId> save(BucketName bucketName, ByteSource data, StoragePolicy storagePolicy);
+
default Publisher<BlobId> save(BucketName bucketName, String data, StoragePolicy storagePolicy) {
return save(bucketName, data.getBytes(StandardCharsets.UTF_8), storagePolicy);
}
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
index d145040..88a0015 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/HashBlobId.java
@@ -19,12 +19,15 @@
package org.apache.james.blob.api;
+import java.io.IOException;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.MoreObjects;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
public class HashBlobId implements BlobId {
@@ -36,6 +39,15 @@ public class HashBlobId implements BlobId {
}
@Override
+ public BlobId forPayload(ByteSource payload) {
+ try {
+ return new HashBlobId(payload.hash(Hashing.sha256()).toString());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
public HashBlobId from(String id) {
Preconditions.checkArgument(!Strings.isNullOrEmpty(id));
return new HashBlobId(id);
diff --git a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java
index bc13d6e..8c57fec 100644
--- a/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java
+++ b/server/blob/blob-api/src/main/java/org/apache/james/blob/api/MetricableBlobStore.java
@@ -26,6 +26,8 @@ import javax.inject.Named;
import org.apache.james.metrics.api.MetricFactory;
import org.reactivestreams.Publisher;
+import com.google.common.io.ByteSource;
+
public class MetricableBlobStore implements BlobStore {
public static final String BLOB_STORE_IMPLEMENTATION = "blobStoreImplementation";
@@ -59,6 +61,11 @@ public class MetricableBlobStore implements BlobStore {
}
@Override
+ public Publisher<BlobId> save(BucketName bucketName, ByteSource data, StoragePolicy storagePolicy) {
+ return metricFactory.decoratePublisherWithTimerMetric(SAVE_INPUT_STREAM_TIMER_NAME, blobStoreImpl.save(bucketName, data, storagePolicy));
+ }
+
+ @Override
public Publisher<byte[]> readBytes(BucketName bucketName, BlobId blobId) {
return metricFactory.decoratePublisherWithTimerMetric(READ_BYTES_TIMER_NAME, blobStoreImpl.readBytes(bucketName, blobId));
}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
index 92f9063..3f6c474 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/BlobStoreContract.java
@@ -36,6 +36,7 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import com.google.common.base.Strings;
+import com.google.common.io.ByteSource;
import reactor.core.publisher.Mono;
@@ -80,6 +81,16 @@ public interface BlobStoreContract extends BucketBlobStoreContract {
@ParameterizedTest
@MethodSource("storagePolicies")
+ default void saveShouldThrowWhenNullByteSource(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ assertThatThrownBy(() -> Mono.from(store.save(defaultBucketName, (ByteSource) null, storagePolicy)).block())
+ .isInstanceOf(NullPointerException.class);
+ }
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
default void saveShouldThrowWhenNullInputStream(BlobStore.StoragePolicy storagePolicy) {
BlobStore store = testee();
BucketName defaultBucketName = store.getDefaultBucketName();
@@ -103,6 +114,19 @@ public interface BlobStoreContract extends BucketBlobStoreContract {
@ParameterizedTest
@MethodSource("storagePolicies")
+ default void saveShouldSaveEmptyByteSource(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, ByteSource.wrap(EMPTY_BYTEARRAY), storagePolicy)).block();
+
+ byte[] bytes = Mono.from(store.readBytes(defaultBucketName, blobId)).block();
+
+ assertThat(new String(bytes, StandardCharsets.UTF_8)).isEmpty();
+ }
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
default void saveShouldSaveEmptyString(BlobStore.StoragePolicy storagePolicy) {
BlobStore store = testee();
BucketName defaultBucketName = store.getDefaultBucketName();
@@ -151,6 +175,19 @@ public interface BlobStoreContract extends BucketBlobStoreContract {
@ParameterizedTest
@MethodSource("storagePolicies")
+ default void readBytesShouldReturnSavedByteSource(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, ByteSource.wrap(SHORT_BYTEARRAY), storagePolicy)).block();
+
+ byte[] bytes = Mono.from(store.readBytes(defaultBucketName, blobId)).block();
+
+ assertThat(bytes).isEqualTo(SHORT_BYTEARRAY);
+ }
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
default void readBytesShouldReturnLongSavedData(BlobStore.StoragePolicy storagePolicy) {
BlobStore store = testee();
BucketName defaultBucketName = store.getDefaultBucketName();
@@ -164,6 +201,19 @@ public interface BlobStoreContract extends BucketBlobStoreContract {
@ParameterizedTest
@MethodSource("storagePolicies")
+ default void readBytesShouldReturnLongSavedByteSource(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, ByteSource.wrap(ELEVEN_KILOBYTES), storagePolicy)).block();
+
+ byte[] bytes = Mono.from(store.readBytes(defaultBucketName, blobId)).block();
+
+ assertThat(bytes).isEqualTo(ELEVEN_KILOBYTES);
+ }
+
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
default void readBytesShouldReturnBigSavedData(BlobStore.StoragePolicy storagePolicy) {
BlobStore store = testee();
BucketName defaultBucketName = store.getDefaultBucketName();
@@ -175,6 +225,19 @@ public interface BlobStoreContract extends BucketBlobStoreContract {
assertThat(bytes).isEqualTo(TWELVE_MEGABYTES);
}
+ @ParameterizedTest
+ @MethodSource("storagePolicies")
+ default void readBytesShouldReturnBigSavedByteSource(BlobStore.StoragePolicy storagePolicy) {
+ BlobStore store = testee();
+ BucketName defaultBucketName = store.getDefaultBucketName();
+
+ BlobId blobId = Mono.from(store.save(defaultBucketName, ByteSource.wrap(TWELVE_MEGABYTES), storagePolicy)).block();
+
+ byte[] bytes = Mono.from(store.readBytes(defaultBucketName, blobId)).block();
+
+ assertThat(bytes).isEqualTo(TWELVE_MEGABYTES);
+ }
+
@Test
default void readShouldThrowWhenNoExistingStream() {
BlobStore store = testee();
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
index 5b4359e..97d8eaa 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/HashBlobIdTest.java
@@ -61,7 +61,7 @@ public class HashBlobIdTest {
@Test
public void forPayloadShouldThrowOnNull() {
- assertThatThrownBy(() -> BLOB_ID_FACTORY.forPayload(null))
+ assertThatThrownBy(() -> BLOB_ID_FACTORY.forPayload((byte[]) null))
.isInstanceOf(IllegalArgumentException.class);
}
diff --git a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java
index 19e6483..848dac1 100644
--- a/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java
+++ b/server/blob/blob-api/src/test/java/org/apache/james/blob/api/TestBlobId.java
@@ -23,6 +23,8 @@ import java.util.Objects;
import org.apache.commons.lang3.NotImplementedException;
+import com.google.common.io.ByteSource;
+
public class TestBlobId implements BlobId {
public static class Factory implements BlobId.Factory {
@@ -32,6 +34,11 @@ public class TestBlobId implements BlobId {
}
@Override
+ public BlobId forPayload(ByteSource payload) {
+ throw new NotImplementedException("Use from(String) instead");
+ }
+
+ @Override
public BlobId from(String id) {
return new TestBlobId(id);
}
diff --git a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java
index a5611d5..bf4a1a0 100644
--- a/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java
+++ b/server/blob/blob-cassandra/src/main/java/org/apache/james/blob/cassandra/cache/CachedBlobStore.java
@@ -40,7 +40,9 @@ import org.apache.james.metrics.api.Metric;
import org.apache.james.metrics.api.MetricFactory;
import org.reactivestreams.Publisher;
+import com.github.fge.lambdas.Throwing;
import com.google.common.base.Preconditions;
+import com.google.common.io.ByteSource;
import reactor.core.publisher.Mono;
@@ -211,6 +213,27 @@ public class CachedBlobStore implements BlobStore {
}
@Override
+ public Publisher<BlobId> save(BucketName bucketName, ByteSource byteSource, StoragePolicy storagePolicy) {
+ Preconditions.checkNotNull(byteSource, "ByteSource must not be null");
+
+ if (isAbleToCache(bucketName, storagePolicy)) {
+ return saveInCache(bucketName, byteSource, storagePolicy);
+ }
+
+ return backend.save(bucketName, byteSource, storagePolicy);
+ }
+
+ private Mono<BlobId> saveInCache(BucketName bucketName, ByteSource byteSource, StoragePolicy storagePolicy) {
+ return Mono.from(backend.save(bucketName, byteSource, storagePolicy))
+ .flatMap(Throwing.function(blobId ->
+ ReadAheadInputStream.eager().of(byteSource.openStream())
+ .length(sizeThresholdInBytes)
+ .firstBytes
+ .map(bytes -> Mono.from(cache.cache(blobId, bytes)).thenReturn(blobId))
+ .orElse(Mono.just(blobId))));
+ }
+
+ @Override
public BucketName getDefaultBucketName() {
return backend.getDefaultBucketName();
}
diff --git a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java
index f4e83c0..f22c6a0 100644
--- a/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java
+++ b/server/blob/blob-cassandra/src/test/java/org/apache/james/blob/cassandra/cache/CachedBlobStoreTest.java
@@ -62,6 +62,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import com.google.common.base.Strings;
+import com.google.common.io.ByteSource;
import reactor.core.publisher.Mono;
@@ -172,6 +173,16 @@ public class CachedBlobStoreTest implements BlobStoreContract {
}
@Test
+ public void shouldCacheWhenEmptyByteSource() {
+ BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, ByteSource.wrap(EMPTY_BYTEARRAY), SIZE_BASED)).block();
+
+ SoftAssertions.assertSoftly(soflty -> {
+ soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block())).hasSameContentAs(new ByteArrayInputStream(EMPTY_BYTEARRAY));
+ soflty.assertThat(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()).containsExactly(EMPTY_BYTEARRAY);
+ });
+ }
+
+ @Test
public void shouldNotCacheWhenEmptyByteArray() {
BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, EMPTY_BYTEARRAY, SIZE_BASED)).block();
@@ -194,6 +205,18 @@ public class CachedBlobStoreTest implements BlobStoreContract {
}
@Test
+ public void shouldCacheWhenFiveKilobytesByteSource() {
+ BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, ByteSource.wrap(APPROXIMATELY_FIVE_KILOBYTES), SIZE_BASED)).block();
+
+ SoftAssertions.assertSoftly(soflty -> {
+ soflty.assertThat(new ByteArrayInputStream(Mono.from(cache.read(blobId)).block()))
+ .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES));
+ soflty.assertThat(new ByteArrayInputStream(Mono.from(backend.readBytes(DEFAULT_BUCKETNAME, blobId)).block()))
+ .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES));
+ });
+ }
+
+ @Test
public void shouldCacheWhenFiveKilobytesBytes() {
BlobId blobId = Mono.from(testee().save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block();
@@ -293,6 +316,30 @@ public class CachedBlobStoreTest implements BlobStoreContract {
}
@Test
+ public void shouldNotCacheWhenReadWithOutDefaultBucketByteSource() {
+ BlobId blobId = Mono.from(backend.save(TEST_BUCKETNAME, ByteSource.wrap(APPROXIMATELY_FIVE_KILOBYTES), SIZE_BASED)).block();
+
+ SoftAssertions.assertSoftly(soflty -> {
+ soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty();
+ soflty.assertThat(testee().read(TEST_BUCKETNAME, blobId, HIGH_PERFORMANCE))
+ .hasSameContentAs(new ByteArrayInputStream(APPROXIMATELY_FIVE_KILOBYTES));
+ soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty();
+ });
+ }
+
+ @Test
+ public void shouldNotCacheWhenReadWithBigByteSource() {
+ BlobId blobId = Mono.from(backend.save(DEFAULT_BUCKETNAME, ByteSource.wrap(TWELVE_MEGABYTES), SIZE_BASED)).block();
+
+ SoftAssertions.assertSoftly(soflty -> {
+ soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty();
+ soflty.assertThat(new ByteArrayInputStream(Mono.from(testee().readBytes(DEFAULT_BUCKETNAME, blobId, HIGH_PERFORMANCE)).block()))
+ .hasSameContentAs(new ByteArrayInputStream(TWELVE_MEGABYTES));
+ soflty.assertThat(Mono.from(cache.read(blobId)).blockOptional()).isEmpty();
+ });
+ }
+
+ @Test
public void readShouldNotPropagateFailures(CassandraCluster cassandra) {
BlobId blobId = Mono.from(testee.save(DEFAULT_BUCKETNAME, APPROXIMATELY_FIVE_KILOBYTES, SIZE_BASED)).block();
diff --git a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
index a32bdff..f37de9d 100644
--- a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
+++ b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/DeDuplicationBlobStore.scala
@@ -21,10 +21,10 @@ package org.apache.james.server.blob.deduplication
import java.io.InputStream
import java.util.concurrent.Callable
+
import com.google.common.base.Preconditions
import com.google.common.hash.{Hashing, HashingInputStream}
import com.google.common.io.{ByteSource, FileBackedOutputStream}
-
import javax.inject.{Inject, Named}
import org.apache.commons.io.IOUtils
import org.apache.james.blob.api.{BlobId, BlobStore, BlobStoreDAO, BucketName}
@@ -55,6 +55,16 @@ class DeDuplicationBlobStore @Inject()(blobStoreDAO: BlobStoreDAO,
.`then`(SMono.just(blobId))
}
+ override def save(bucketName: BucketName, data: ByteSource, storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = {
+ Preconditions.checkNotNull(bucketName)
+ Preconditions.checkNotNull(data)
+
+ val blobId = blobIdFactory.forPayload(data)
+
+ SMono(blobStoreDAO.save(bucketName, blobId, data))
+ .`then`(SMono.just(blobId))
+ }
+
override def save(bucketName: BucketName, data: InputStream, storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = {
Preconditions.checkNotNull(bucketName)
Preconditions.checkNotNull(data)
diff --git a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
index e7e5020..b4a0f14 100644
--- a/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
+++ b/server/blob/blob-storage-strategy/src/main/scala/org/apache/james/server/blob/deduplication/PassThroughBlobStore.scala
@@ -22,6 +22,7 @@ package org.apache.james.server.blob.deduplication
import java.io.InputStream
import com.google.common.base.Preconditions
+import com.google.common.io.ByteSource
import javax.inject.{Inject, Named}
import org.apache.james.blob.api.{BlobId, BlobStore, BlobStoreDAO, BucketName}
import org.reactivestreams.Publisher
@@ -51,6 +52,15 @@ class PassThroughBlobStore @Inject()(blobStoreDAO: BlobStoreDAO,
.`then`(SMono.just(blobId))
}
+ override def save(bucketName: BucketName, data: ByteSource, storagePolicy: BlobStore.StoragePolicy): Publisher[BlobId] = {
+ Preconditions.checkNotNull(bucketName)
+ Preconditions.checkNotNull(data)
+ val blobId = blobIdFactory.randomId()
+
+ SMono(blobStoreDAO.save(bucketName, blobId, data))
+ .`then`(SMono.just(blobId))
+ }
+
override def readBytes(bucketName: BucketName, blobId: BlobId): Publisher[Array[Byte]] = {
Preconditions.checkNotNull(bucketName)
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org