You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by rc...@apache.org on 2024/11/21 03:08:32 UTC
(james-project) branch master updated (a156877b5e -> 34ca224bad)
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
from a156877b5e JAMES-2182 Extract PathConverterTest to Contract class
new 5ed599e4ed JAMES-4085 Introduce S3MinioExtension - support https
new 3eaef14ce3 JAMES-4085 [S3 SSEC] - S3BlobStoreDAO - extract build Get/Put ObjectRequestBuilder
new 34ca224bad JAMES-4085 [S3 SSEC] SSEC basic implementation
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
server/blob/blob-s3/pom.xml | 6 +
.../blob/objectstorage/aws/S3BlobStoreDAO.java | 147 +++++++++++++--------
.../blob/objectstorage/aws/S3RequestOption.java} | 23 ++--
.../aws/sse/S3SSECConfiguration.java} | 29 ++--
.../aws/sse/S3SSECustomerKeyFactory.java | 61 +++++++++
.../aws/sse/S3SSECustomerKeyGenerator.java | 55 ++++++++
.../blob/objectstorage/aws/S3MinioDocker.java | 109 +++++++++++++++
.../blob/objectstorage/aws/S3MinioExtension.java} | 29 ++--
.../james/blob/objectstorage/aws/S3MinioTest.java | 40 +-----
.../S3BlobStoreDAOWithSSECTest.java} | 107 +++++----------
.../blob/objectstorage/aws/sse/S3SSECContract.java | 89 +++++++++++++
.../aws/sse/S3SSECustomerKeyGeneratorTest.java | 61 +++++++++
.../aws/sse/SingleCustomerKeyFactoryTest.java | 65 +++++++++
.../blob-s3/src/test/resources/minio/private.key | 5 +
.../blob-s3/src/test/resources/minio/public.crt | 14 ++
15 files changed, 636 insertions(+), 204 deletions(-)
copy server/blob/{blob-export-api/src/main/java/org/apache/james/blob/export/api/ExportedFileNamesGenerator.java => blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3RequestOption.java} (65%)
copy server/{protocols/jmap-rfc-8621/src/main/java/org/apache/james/jmap/mailet/filter/RuleMatcher.java => blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECConfiguration.java} (60%)
create mode 100644 server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyFactory.java
create mode 100644 server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGenerator.java
create mode 100644 server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java
copy server/{apps/distributed-app/src/test/java/org/apache/james/modules/RabbitMQExtension.java => blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java} (75%)
copy server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/{S3MinioTest.java => sse/S3BlobStoreDAOWithSSECTest.java} (52%)
create mode 100644 server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java
create mode 100644 server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGeneratorTest.java
create mode 100644 server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/SingleCustomerKeyFactoryTest.java
create mode 100644 server/blob/blob-s3/src/test/resources/minio/private.key
create mode 100644 server/blob/blob-s3/src/test/resources/minio/public.crt
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 03/03: JAMES-4085 [S3 SSEC] SSEC basic implementation
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 34ca224bade939856d82a45986ba53ab63c2fd78
Author: TungTV <vt...@linagora.com>
AuthorDate: Tue Nov 19 10:17:14 2024 +0700
JAMES-4085 [S3 SSEC] SSEC basic implementation
---
.../blob/objectstorage/aws/S3BlobStoreDAO.java | 44 ++++++--
.../blob/objectstorage/aws/S3RequestOption.java | 38 +++++++
.../objectstorage/aws/sse/S3SSECConfiguration.java | 46 ++++++++
.../aws/sse/S3SSECustomerKeyFactory.java | 61 +++++++++++
.../aws/sse/S3SSECustomerKeyGenerator.java | 55 ++++++++++
.../aws/sse/S3BlobStoreDAOWithSSECTest.java | 120 +++++++++++++++++++++
.../blob/objectstorage/aws/sse/S3SSECContract.java | 89 +++++++++++++++
.../aws/sse/S3SSECustomerKeyGeneratorTest.java | 61 +++++++++++
.../aws/sse/SingleCustomerKeyFactoryTest.java | 65 +++++++++++
9 files changed, 572 insertions(+), 7 deletions(-)
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
index 9b4b993de3..8c34c3a6b6 100644
--- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
@@ -114,14 +114,23 @@ public class S3BlobStoreDAO implements BlobStoreDAO {
private final S3AsyncClient client;
private final S3BlobStoreConfiguration configuration;
private final BlobId.Factory blobIdFactory;
+ private final S3RequestOption s3RequestOption;
@Inject
public S3BlobStoreDAO(S3ClientFactory s3ClientFactory,
- S3BlobStoreConfiguration configuration,
- BlobId.Factory blobIdFactory) {
+ S3BlobStoreConfiguration configuration,
+ BlobId.Factory blobIdFactory) {
+ this(s3ClientFactory, configuration, blobIdFactory, S3RequestOption.DEFAULT);
+ }
+
+ public S3BlobStoreDAO(S3ClientFactory s3ClientFactory,
+ S3BlobStoreConfiguration configuration,
+ BlobId.Factory blobIdFactory,
+ S3RequestOption s3RequestOption) {
this.configuration = configuration;
this.client = s3ClientFactory.get();
this.blobIdFactory = blobIdFactory;
+ this.s3RequestOption = s3RequestOption;
bucketNameResolver = BucketNameResolver.builder()
.prefix(configuration.getBucketPrefix())
@@ -216,9 +225,20 @@ public class S3BlobStoreDAO implements BlobStoreDAO {
}
private Mono<GetObjectRequest.Builder> buildGetObjectRequestBuilder(BucketName bucketName, BlobId blobId) {
- return Mono.just(GetObjectRequest.builder()
+ GetObjectRequest.Builder baseBuilder = GetObjectRequest.builder()
.bucket(bucketName.asString())
- .key(blobId.asString()));
+ .key(blobId.asString());
+
+ if (s3RequestOption.ssec().enable()) {
+ return Mono.from(s3RequestOption.ssec().sseCustomerKeyFactory().get()
+ .generate(bucketName, blobId))
+ .map(sseCustomerKey -> baseBuilder
+ .sseCustomerAlgorithm(sseCustomerKey.ssecAlgorithm())
+ .sseCustomerKey(sseCustomerKey.customerKey())
+ .sseCustomerKeyMD5(sseCustomerKey.md5()));
+ }
+
+ return Mono.just(baseBuilder);
}
@Override
@@ -279,10 +299,20 @@ public class S3BlobStoreDAO implements BlobStoreDAO {
}
private Mono<PutObjectRequest.Builder> buildPutObjectRequestBuilder(BucketName bucketName, long contentLength, BlobId blobId) {
- return Mono.just(PutObjectRequest.builder()
+ PutObjectRequest.Builder baseBuilder = PutObjectRequest.builder()
.bucket(bucketName.asString())
- .contentLength(contentLength)
- .key(blobId.asString()));
+ .key(blobId.asString())
+ .contentLength(contentLength);
+
+ if (s3RequestOption.ssec().enable()) {
+ return Mono.from(s3RequestOption.ssec().sseCustomerKeyFactory().get().generate(bucketName, blobId))
+ .map(sseCustomerKey -> baseBuilder
+ .sseCustomerAlgorithm(sseCustomerKey.ssecAlgorithm())
+ .sseCustomerKey(sseCustomerKey.customerKey())
+ .sseCustomerKeyMD5(sseCustomerKey.md5()));
+ }
+
+ return Mono.just(baseBuilder);
}
private Flux<ByteBuffer> chunkStream(int chunkSize, InputStream stream) {
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3RequestOption.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3RequestOption.java
new file mode 100644
index 0000000000..5de59834d2
--- /dev/null
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3RequestOption.java
@@ -0,0 +1,38 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws;
+
+import java.util.Optional;
+
+import org.apache.james.blob.objectstorage.aws.sse.S3SSECustomerKeyFactory;
+
+import com.google.common.base.Preconditions;
+
+public record S3RequestOption(SSEC ssec) {
+ static S3RequestOption DEFAULT = new S3RequestOption(S3RequestOption.SSEC.DISABLED);
+
+ public record SSEC(boolean enable, java.util.Optional<S3SSECustomerKeyFactory> sseCustomerKeyFactory) {
+ static S3RequestOption.SSEC DISABLED = new S3RequestOption.SSEC(false, Optional.empty());
+
+ public SSEC {
+ Preconditions.checkArgument(!enable || sseCustomerKeyFactory.isPresent(), "SSE Customer Key Factory must be present when SSE is enabled");
+ }
+ }
+}
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECConfiguration.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECConfiguration.java
new file mode 100644
index 0000000000..9160fa8f70
--- /dev/null
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECConfiguration.java
@@ -0,0 +1,46 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import java.util.List;
+import java.util.Optional;
+
+import com.google.common.base.Preconditions;
+
+public interface S3SSECConfiguration {
+
+ String SSEC_ALGORITHM_DEFAULT = "AES256";
+ String CUSTOMER_KEY_FACTORY_ALGORITHM_DEFAULT = "PBKDF2WithHmacSHA256";
+ List<String> SUPPORTED_ALGORITHMS = List.of(SSEC_ALGORITHM_DEFAULT);
+
+ String ssecAlgorithm();
+
+ default Optional<String> customerKeyFactoryAlgorithm() {
+ return Optional.of(CUSTOMER_KEY_FACTORY_ALGORITHM_DEFAULT);
+ }
+
+ record Basic(String ssecAlgorithm,
+ String masterPassword,
+ String salt) implements S3SSECConfiguration {
+ public Basic {
+ Preconditions.checkArgument(SUPPORTED_ALGORITHMS.contains(ssecAlgorithm), "Unsupported algorithm: " + ssecAlgorithm + ". The supported algorithms are: " + SUPPORTED_ALGORITHMS);
+ }
+ }
+}
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyFactory.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyFactory.java
new file mode 100644
index 0000000000..f694051f92
--- /dev/null
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyFactory.java
@@ -0,0 +1,61 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
+
+import org.apache.james.blob.api.BlobId;
+import org.apache.james.blob.api.BucketName;
+import org.reactivestreams.Publisher;
+
+import com.github.fge.lambdas.Throwing;
+
+import reactor.core.publisher.Mono;
+
+public interface S3SSECustomerKeyFactory {
+
+ record SSECustomerKey(String customerKey,
+ String md5,
+ String ssecAlgorithm) {
+ }
+
+ Publisher<SSECustomerKey> generate(BucketName bucketName, BlobId blobId);
+
+ class SingleCustomerKeyFactory implements S3SSECustomerKeyFactory {
+
+ private final SSECustomerKey sseCustomerKey;
+
+ public SingleCustomerKeyFactory(S3SSECConfiguration.Basic sseCustomerConfiguration) throws InvalidKeySpecException, NoSuchAlgorithmException {
+ S3SSECustomerKeyGenerator sseCustomerKeyGenerator = sseCustomerConfiguration.customerKeyFactoryAlgorithm()
+ .map(Throwing.function(S3SSECustomerKeyGenerator::new))
+ .orElseGet(Throwing.supplier(S3SSECustomerKeyGenerator::new));
+
+ String customerKey = sseCustomerKeyGenerator.generateCustomerKey(sseCustomerConfiguration.masterPassword(), sseCustomerConfiguration.salt());
+ String customerMd5 = sseCustomerKeyGenerator.generateCustomerKeyMd5(customerKey);
+ this.sseCustomerKey = new SSECustomerKey(customerKey, customerMd5, sseCustomerConfiguration.ssecAlgorithm());
+ }
+
+ @Override
+ public Mono<SSECustomerKey> generate(BucketName bucketName, BlobId blobId) {
+ return Mono.just(sseCustomerKey);
+ }
+ }
+}
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGenerator.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGenerator.java
new file mode 100644
index 0000000000..d712ac8752
--- /dev/null
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGenerator.java
@@ -0,0 +1,55 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
+import java.util.Base64;
+
+import javax.crypto.SecretKeyFactory;
+import javax.crypto.spec.PBEKeySpec;
+
+import com.google.common.hash.Hashing;
+
+public class S3SSECustomerKeyGenerator {
+
+ public static final int ITERATION_COUNT = 65536;
+ public static final int KEY_LENGTH = 256;
+
+ private final SecretKeyFactory secretKeyFactory;
+
+ public S3SSECustomerKeyGenerator() throws NoSuchAlgorithmException {
+ this(S3SSECConfiguration.CUSTOMER_KEY_FACTORY_ALGORITHM_DEFAULT);
+ }
+
+ public S3SSECustomerKeyGenerator(String customerKeyFactoryAlgorithm) throws NoSuchAlgorithmException {
+ this.secretKeyFactory = SecretKeyFactory.getInstance(customerKeyFactoryAlgorithm);
+ }
+
+ public String generateCustomerKey(String masterPassword, String salt) throws InvalidKeySpecException {
+ PBEKeySpec spec = new PBEKeySpec(masterPassword.toCharArray(), salt.getBytes(), ITERATION_COUNT, KEY_LENGTH);
+ byte[] derivedKey = secretKeyFactory.generateSecret(spec).getEncoded();
+ return Base64.getEncoder().encodeToString(derivedKey);
+ }
+
+ public String generateCustomerKeyMd5(String customerKey) {
+ return Base64.getEncoder().encodeToString(Hashing.md5().hashBytes(Base64.getDecoder().decode(customerKey)).asBytes());
+ }
+}
\ No newline at end of file
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java
new file mode 100644
index 0000000000..4c55a699fa
--- /dev/null
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3BlobStoreDAOWithSSECTest.java
@@ -0,0 +1,120 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME;
+import static org.apache.james.blob.objectstorage.aws.JamesS3MetricPublisher.DEFAULT_S3_METRICS_PREFIX;
+import static org.apache.james.blob.objectstorage.aws.S3BlobStoreConfiguration.UPLOAD_RETRY_EXCEPTION_PREDICATE;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.james.blob.api.BlobStoreDAO;
+import org.apache.james.blob.api.BlobStoreDAOContract;
+import org.apache.james.blob.api.TestBlobId;
+import org.apache.james.blob.objectstorage.aws.JamesS3MetricPublisher;
+import org.apache.james.blob.objectstorage.aws.Region;
+import org.apache.james.blob.objectstorage.aws.S3BlobStoreConfiguration;
+import org.apache.james.blob.objectstorage.aws.S3BlobStoreDAO;
+import org.apache.james.blob.objectstorage.aws.S3ClientFactory;
+import org.apache.james.blob.objectstorage.aws.S3MinioExtension;
+import org.apache.james.blob.objectstorage.aws.S3RequestOption;
+import org.apache.james.metrics.api.NoopGaugeRegistry;
+import org.apache.james.metrics.tests.RecordingMetricFactory;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import reactor.core.publisher.Flux;
+import reactor.util.retry.Retry;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+
+public class S3BlobStoreDAOWithSSECTest implements BlobStoreDAOContract, S3SSECContract {
+
+ @RegisterExtension
+ static S3MinioExtension minoExtension = new S3MinioExtension();
+
+ private static S3BlobStoreDAO testee;
+ private static S3ClientFactory s3ClientFactory;
+
+ @BeforeAll
+ static void setUp() throws Exception {
+ S3BlobStoreConfiguration s3Configuration = S3BlobStoreConfiguration.builder()
+ .authConfiguration(minoExtension.minioDocker().getAwsS3AuthConfiguration())
+ .region(Region.of(software.amazon.awssdk.regions.Region.EU_WEST_1.id()))
+ .uploadRetrySpec(Optional.of(Retry.backoff(3, java.time.Duration.ofSeconds(1))
+ .filter(UPLOAD_RETRY_EXCEPTION_PREDICATE)))
+ .build();
+
+ s3ClientFactory = new S3ClientFactory(s3Configuration, () -> new JamesS3MetricPublisher(new RecordingMetricFactory(), new NoopGaugeRegistry(),
+ DEFAULT_S3_METRICS_PREFIX));
+
+ S3SSECustomerKeyFactory sseCustomerKeyFactory = new S3SSECustomerKeyFactory.SingleCustomerKeyFactory(new S3SSECConfiguration.Basic("AES256", "masterPassword", "salt"));
+
+ S3RequestOption s3RequestOption = new S3RequestOption(new S3RequestOption.SSEC(true, Optional.of(sseCustomerKeyFactory)));
+ testee = new S3BlobStoreDAO(s3ClientFactory, s3Configuration, new TestBlobId.Factory(), s3RequestOption);
+ }
+
+ @BeforeEach
+ void beforeEach() throws Exception {
+ // Why? https://github.com/apache/james-project/pull/1981#issuecomment-2380396460
+ s3ClientFactory.get().createBucket(builder -> builder.bucket(TEST_BUCKET_NAME.asString()))
+ .get();
+ }
+
+ @Override
+ public BlobStoreDAO testee() {
+ return testee;
+ }
+
+ @Override
+ public S3AsyncClient s3Client() {
+ return s3ClientFactory.get();
+ }
+
+ private void deleteBucket(String bucketName) {
+ try {
+ s3ClientFactory.get().deleteBucket(builder -> builder.bucket(bucketName))
+ .get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw new RuntimeException("Error while deleting bucket", e);
+ }
+ }
+
+ @Test
+ @Override
+ public void listBucketsShouldReturnEmptyWhenNone() {
+ deleteBucket(TEST_BUCKET_NAME.asString());
+
+ BlobStoreDAO store = testee();
+
+ assertThat(Flux.from(store.listBuckets()).collectList().block())
+ .isEmpty();
+ }
+
+ @Test
+ @Override
+ @Disabled("S3minio return `Connection: close` in header response, https://github.com/apache/james-project/pull/1981#issuecomment-2380396460")
+ public void deleteBucketConcurrentlyShouldNotFail() {
+ }
+}
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java
new file mode 100644
index 0000000000..f18033746f
--- /dev/null
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECContract.java
@@ -0,0 +1,89 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import static org.apache.james.blob.api.BlobStoreDAOFixture.SHORT_BYTEARRAY;
+import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BLOB_ID;
+import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.james.blob.api.BlobStoreDAO;
+import org.junit.jupiter.api.Test;
+
+import reactor.core.publisher.Mono;
+import software.amazon.awssdk.core.BytesWrapper;
+import software.amazon.awssdk.core.async.AsyncResponseTransformer;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+
+public interface S3SSECContract {
+
+ BlobStoreDAO testee();
+
+ S3AsyncClient s3Client();
+
+ @Test
+ default void getObjectShouldFailWhenNotDefineCustomerKeyInHeaderRequest() {
+ Mono.from(testee().save(TEST_BUCKET_NAME, TEST_BLOB_ID, SHORT_BYTEARRAY)).block();
+
+ assertThatThrownBy(() -> s3Client().getObject(GetObjectRequest.builder()
+ .bucket(TEST_BUCKET_NAME.asString())
+ .key(TEST_BLOB_ID.asString())
+ .build(), AsyncResponseTransformer.toBytes())
+ .thenApply(BytesWrapper::asByteArray)
+ .get())
+ .hasMessageContaining("The object was stored using a form of Server Side Encryption");
+ }
+
+ @Test
+ default void getObjectShouldFailWhenInvalidCustomerKeyInRequest() {
+ Mono.from(testee().save(TEST_BUCKET_NAME, TEST_BLOB_ID, SHORT_BYTEARRAY)).block();
+
+ assertThatThrownBy(() -> s3Client().getObject(GetObjectRequest.builder()
+ .bucket(TEST_BUCKET_NAME.asString())
+ .key(TEST_BLOB_ID.asString())
+ .sseCustomerKey("invalid")
+ .sseCustomerKeyMD5("123")
+ .build(), AsyncResponseTransformer.toBytes())
+ .thenApply(BytesWrapper::asByteArray)
+ .get())
+ .hasMessageContaining("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm");
+ }
+
+ @Test
+ default void getObjectShouldFailWhenNotMatchedCustomerKeyInRequest() throws Exception {
+ Mono.from(testee().save(TEST_BUCKET_NAME, TEST_BLOB_ID, SHORT_BYTEARRAY)).block();
+ S3SSECustomerKeyGenerator sseCustomerKeyGenerator = new S3SSECustomerKeyGenerator();
+
+ String customerKey = sseCustomerKeyGenerator.generateCustomerKey("random1", "salt123random");
+ String customerKeyMd5 = sseCustomerKeyGenerator.generateCustomerKeyMd5(customerKey);
+
+ assertThatThrownBy(() -> s3Client().getObject(GetObjectRequest.builder()
+ .bucket(TEST_BUCKET_NAME.asString())
+ .key(TEST_BLOB_ID.asString())
+ .sseCustomerKey(customerKey)
+ .sseCustomerKeyMD5(customerKeyMd5)
+ .sseCustomerAlgorithm("AES256")
+ .build(), AsyncResponseTransformer.toBytes())
+ .thenApply(BytesWrapper::asByteArray)
+ .get())
+ .hasMessageContaining("Access Denied");
+ }
+}
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGeneratorTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGeneratorTest.java
new file mode 100644
index 0000000000..27705798cc
--- /dev/null
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/S3SSECustomerKeyGeneratorTest.java
@@ -0,0 +1,61 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import java.time.Duration;
+import java.util.Map;
+
+import org.apache.james.util.concurrency.ConcurrentTestRunner;
+import org.junit.jupiter.api.Test;
+
+import com.google.common.collect.ImmutableMap;
+
+public class S3SSECustomerKeyGeneratorTest {
+ record SampleTest(String masterPassword, String salt, String expectedCustomerKey) {
+ }
+
+ static Map<Integer, SampleTest> SAMPLE_TEST = ImmutableMap.of(
+ 1, new SampleTest("test1", "salt1", "Le39M/h9Wi72qMMV9O4ibqZxWb9cL3rQRUy0rfMdODc="),
+ 2, new SampleTest("test2", "salt2", "Yo8T24EXwAZm3lyGyzfiyYx0FcSdK5ai9pmgO6yLDe8="),
+ 3, new SampleTest("test3", "salt3", "KGMvLVMB0Y7U2WpYiQmbWn+g9fE3ZvH0pSrx1BzNups="),
+ 4, new SampleTest("test4", "salt4", "kGbGrfAgfAjkEByylQoFO77yR0JeSA44fhnkuREdrkY="),
+ 5, new SampleTest("test5", "salt5", "dxaGCxA7c7bmG2gEjHkfLMvH6SFUQp0d0wa0xKuia3Q="),
+ 6, new SampleTest("test6", "salt6", "GhNsbP2E/2uSMpgD59jR1W8oTmoRwZ/bo1ScSqh4wiQ="),
+ 7, new SampleTest("test7", "salt7", "X2VE+qpQnTFnWN5e/SgQyoyBXfxVM3Oy2+2f8Auba6o="),
+ 8, new SampleTest("test8", "salt8", "EVz2OnBEOyfuH4h4p5X7utZVJ+LXAojnti102Wzp6Lc="),
+ 9, new SampleTest("test9", "salt9", "HNA8IazLNDFGATKjBjzeNJPZ6fn9/g19rzIFAHYVbGs="),
+ 10, new SampleTest("test10", "salt10", "8R4WuFhTDpRMsQjht6XMrq4CFODC9oRZ/zucRXZ9+JU="));
+
+ @Test
+ void generateShouldWorkCorrectWhenConcurrent() throws Exception {
+ S3SSECustomerKeyGenerator testee = new S3SSECustomerKeyGenerator("PBKDF2WithHmacSHA256");
+ ConcurrentTestRunner.builder()
+ .operation(((threadNumber, step) -> {
+ int index = threadNumber + 1;
+ String customerKey = testee.generateCustomerKey("test" + index, "salt" + index);
+ assertThat(customerKey).isEqualTo(SAMPLE_TEST.get(index).expectedCustomerKey());
+ }))
+ .threadCount(10)
+ .operationCount(100)
+ .runSuccessfullyWithin(Duration.ofMinutes(1));
+ }
+}
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/SingleCustomerKeyFactoryTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/SingleCustomerKeyFactoryTest.java
new file mode 100644
index 0000000000..08f878a1f0
--- /dev/null
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/sse/SingleCustomerKeyFactoryTest.java
@@ -0,0 +1,65 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws.sse;
+
+import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+
+import java.security.NoSuchAlgorithmException;
+import java.security.spec.InvalidKeySpecException;
+
+import org.apache.james.blob.api.BucketName;
+import org.apache.james.blob.api.TestBlobId;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import reactor.core.publisher.Mono;
+
+public class SingleCustomerKeyFactoryTest {
+ private static S3SSECustomerKeyFactory.SingleCustomerKeyFactory testee;
+
+ @BeforeAll
+ static void setup() throws InvalidKeySpecException, NoSuchAlgorithmException {
+ testee = new S3SSECustomerKeyFactory.SingleCustomerKeyFactory(new S3SSECConfiguration.Basic("AES256", "masterPassword", "salt"));
+
+ }
+
+ @Test
+ void generateShouldReturnSSECustomerKey() {
+ // When
+ S3SSECustomerKeyFactory.SSECustomerKey sseCustomerKey = Mono.from(testee.generate(BucketName.of("bucket1"), new TestBlobId("blobId"))).block();
+
+ // Then
+ assertThat(sseCustomerKey.customerKey()).isEqualTo("tsY5n0n10fcpjm7aGArqbw231ptLGAF5ubxMxI2+QYw=");
+ assertThat(sseCustomerKey.md5()).isEqualTo("G6IB1YLHiY/9uCh3TJhxzQ==");
+ assertThat(sseCustomerKey.ssecAlgorithm()).isEqualTo("AES256");
+ }
+
+ @Test
+ void generateShouldReturnSameSSECustomerKey() {
+ // When
+ S3SSECustomerKeyFactory.SSECustomerKey sseCustomerKey1 = Mono.from(testee.generate(BucketName.of("bucket1"), new TestBlobId("blobId"))).block();
+ S3SSECustomerKeyFactory.SSECustomerKey sseCustomerKey2 = Mono.from(testee.generate(BucketName.of("bucket1"), new TestBlobId("blobId"))).block();
+ S3SSECustomerKeyFactory.SSECustomerKey sseCustomerKey3 = Mono.from(testee.generate(BucketName.of("bucket3"), new TestBlobId("blobId3"))).block();
+
+ // Then
+ assertThat(sseCustomerKey1).isEqualTo(sseCustomerKey2)
+ .isEqualTo(sseCustomerKey3);
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 02/03: JAMES-4085 [S3 SSEC] - S3BlobStoreDAO - extract build Get/Put ObjectRequestBuilder
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 3eaef14ce3313005fdd0e2ecba6184da4e663a1b
Author: TungTV <vt...@linagora.com>
AuthorDate: Tue Nov 19 10:17:14 2024 +0700
JAMES-4085 [S3 SSEC] - S3BlobStoreDAO - extract build Get/Put ObjectRequestBuilder
---
.../blob/objectstorage/aws/S3BlobStoreDAO.java | 113 +++++++++++----------
1 file changed, 62 insertions(+), 51 deletions(-)
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
index 74ac19a483..9b4b993de3 100644
--- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
@@ -63,12 +63,14 @@ import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.model.Bucket;
import software.amazon.awssdk.services.s3.model.BucketAlreadyOwnedByYouException;
import software.amazon.awssdk.services.s3.model.DeleteObjectsResponse;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.ListBucketsResponse;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.NoSuchBucketException;
import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
import software.amazon.awssdk.services.s3.model.ObjectIdentifier;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
import software.amazon.awssdk.services.s3.model.S3Object;
@@ -166,36 +168,36 @@ public class S3BlobStoreDAO implements BlobStoreDAO {
}
private Mono<FluxResponse> getObject(BucketName bucketName, BlobId blobId) {
- return Mono.fromFuture(() ->
- client.getObject(
- builder -> builder.bucket(bucketName.asString()).key(blobId.asString()),
- new AsyncResponseTransformer<GetObjectResponse, FluxResponse>() {
-
- FluxResponse response;
-
- @Override
- public CompletableFuture<FluxResponse> prepare() {
- response = new FluxResponse();
- return response.supportingCompletableFuture;
- }
-
- @Override
- public void onResponse(GetObjectResponse response) {
- this.response.sdkResponse = response;
- }
-
- @Override
- public void exceptionOccurred(Throwable error) {
- this.response.supportingCompletableFuture.completeExceptionally(error);
- }
-
- @Override
- public void onStream(SdkPublisher<ByteBuffer> publisher) {
- response.flux = Flux.from(publisher);
- response.supportingCompletableFuture.complete(response);
- }
- }))
- .switchIfEmpty(Mono.error(() -> new ObjectStoreIOException("Request was unexpectedly canceled, no GetObjectResponse")));
+ return buildGetObjectRequestBuilder(bucketName, blobId)
+ .flatMap(getObjectRequestBuilder -> Mono.fromFuture(() ->
+ client.getObject(getObjectRequestBuilder.build(),
+ new AsyncResponseTransformer<GetObjectResponse, FluxResponse>() {
+
+ FluxResponse response;
+
+ @Override
+ public CompletableFuture<FluxResponse> prepare() {
+ response = new FluxResponse();
+ return response.supportingCompletableFuture;
+ }
+
+ @Override
+ public void onResponse(GetObjectResponse response) {
+ this.response.sdkResponse = response;
+ }
+
+ @Override
+ public void exceptionOccurred(Throwable error) {
+ this.response.supportingCompletableFuture.completeExceptionally(error);
+ }
+
+ @Override
+ public void onStream(SdkPublisher<ByteBuffer> publisher) {
+ response.flux = Flux.from(publisher);
+ response.supportingCompletableFuture.complete(response);
+ }
+ }))
+ .switchIfEmpty(Mono.error(() -> new ObjectStoreIOException("Request was unexpectedly canceled, no GetObjectResponse"))));
}
@@ -203,27 +205,31 @@ public class S3BlobStoreDAO implements BlobStoreDAO {
public Mono<byte[]> readBytes(BucketName bucketName, BlobId blobId) {
BucketName resolvedBucketName = bucketNameResolver.resolve(bucketName);
- return Mono.fromFuture(() ->
- client.getObject(
- builder -> builder.bucket(resolvedBucketName.asString()).key(blobId.asString()),
- new MinimalCopyBytesResponseTransformer(configuration, blobId)))
- .onErrorMap(NoSuchBucketException.class, e -> new ObjectNotFoundException("Bucket not found " + resolvedBucketName.asString(), e))
- .onErrorMap(NoSuchKeyException.class, e -> new ObjectNotFoundException("Blob not found " + blobId.asString() + " in bucket " + resolvedBucketName.asString(), e))
- .publishOn(Schedulers.parallel())
- .map(BytesWrapper::asByteArrayUnsafe)
- .onErrorMap(e -> e.getCause() instanceof OutOfMemoryError, Throwable::getCause);
+ return buildGetObjectRequestBuilder(resolvedBucketName, blobId)
+ .flatMap(putObjectRequest -> Mono.fromFuture(() ->
+ client.getObject(putObjectRequest.build(), new MinimalCopyBytesResponseTransformer(configuration, blobId)))
+ .onErrorMap(NoSuchBucketException.class, e -> new ObjectNotFoundException("Bucket not found " + resolvedBucketName.asString(), e))
+ .onErrorMap(NoSuchKeyException.class, e -> new ObjectNotFoundException("Blob not found " + blobId.asString() + " in bucket " + resolvedBucketName.asString(), e))
+ .publishOn(Schedulers.parallel())
+ .map(BytesWrapper::asByteArrayUnsafe)
+ .onErrorMap(e -> e.getCause() instanceof OutOfMemoryError, Throwable::getCause));
+ }
+
+ private Mono<GetObjectRequest.Builder> buildGetObjectRequestBuilder(BucketName bucketName, BlobId blobId) {
+ return Mono.just(GetObjectRequest.builder()
+ .bucket(bucketName.asString())
+ .key(blobId.asString()));
}
@Override
public Mono<Void> save(BucketName bucketName, BlobId blobId, byte[] data) {
BucketName resolvedBucketName = bucketNameResolver.resolve(bucketName);
- return Mono.fromFuture(() ->
- client.putObject(
- builder -> builder.bucket(resolvedBucketName.asString()).key(blobId.asString()).contentLength((long) data.length),
- AsyncRequestBody.fromBytes(data)))
- .retryWhen(createBucketOnRetry(resolvedBucketName))
- .publishOn(Schedulers.parallel())
+ return buildPutObjectRequestBuilder(resolvedBucketName, data.length, blobId)
+ .flatMap(putObjectRequest -> Mono.fromFuture(() ->
+ client.putObject(putObjectRequest.build(), AsyncRequestBody.fromBytes(data)))
+ .retryWhen(createBucketOnRetry(resolvedBucketName))
+ .publishOn(Schedulers.parallel()))
.then();
}
@@ -266,12 +272,17 @@ public class S3BlobStoreDAO implements BlobStoreDAO {
private Mono<PutObjectResponse> save(BucketName resolvedBucketName, BlobId blobId, InputStream stream, long contentLength) {
int chunkSize = Math.min((int) contentLength, CHUNK_SIZE);
- return Mono.fromFuture(() -> client.putObject(builder -> builder
- .bucket(resolvedBucketName.asString())
- .contentLength(contentLength)
- .key(blobId.asString()),
- AsyncRequestBody.fromPublisher(chunkStream(chunkSize, stream)
- .subscribeOn(Schedulers.boundedElastic()))));
+ return buildPutObjectRequestBuilder(resolvedBucketName, contentLength, blobId)
+ .flatMap(putObjectRequest -> Mono.fromFuture(() -> client.putObject(putObjectRequest.build(),
+ AsyncRequestBody.fromPublisher(chunkStream(chunkSize, stream)
+ .subscribeOn(Schedulers.boundedElastic())))));
+ }
+
+ private Mono<PutObjectRequest.Builder> buildPutObjectRequestBuilder(BucketName bucketName, long contentLength, BlobId blobId) {
+ return Mono.just(PutObjectRequest.builder()
+ .bucket(bucketName.asString())
+ .contentLength(contentLength)
+ .key(blobId.asString()));
}
private Flux<ByteBuffer> chunkStream(int chunkSize, InputStream stream) {
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org
(james-project) 01/03: JAMES-4085 Introduce S3MinioExtension - support https
Posted by rc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 5ed599e4ed1130dc4b730f1405a5819ed6585c51
Author: TungTV <vt...@linagora.com>
AuthorDate: Tue Nov 19 10:17:14 2024 +0700
JAMES-4085 Introduce S3MinioExtension - support https
---
server/blob/blob-s3/pom.xml | 6 ++
.../blob/objectstorage/aws/S3MinioDocker.java | 109 +++++++++++++++++++++
.../blob/objectstorage/aws/S3MinioExtension.java | 59 +++++++++++
.../james/blob/objectstorage/aws/S3MinioTest.java | 40 ++------
.../blob-s3/src/test/resources/minio/private.key | 5 +
.../blob-s3/src/test/resources/minio/public.crt | 14 +++
6 files changed, 199 insertions(+), 34 deletions(-)
diff --git a/server/blob/blob-s3/pom.xml b/server/blob/blob-s3/pom.xml
index 2de0f6a4c1..01573fdaf7 100644
--- a/server/blob/blob-s3/pom.xml
+++ b/server/blob/blob-s3/pom.xml
@@ -56,6 +56,12 @@
<groupId>${james.groupId}</groupId>
<artifactId>james-core</artifactId>
</dependency>
+ <dependency>
+ <groupId>${james.groupId}</groupId>
+ <artifactId>james-server-guice-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>${james.groupId}</groupId>
<artifactId>james-server-lifecycle-api</artifactId>
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java
new file mode 100644
index 0000000000..54bf3e5a91
--- /dev/null
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioDocker.java
@@ -0,0 +1,109 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws;
+
+import java.time.Duration;
+import java.util.UUID;
+
+import org.apache.http.client.utils.URIBuilder;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.DockerImageName;
+
+import com.github.fge.lambdas.Throwing;
+import com.google.common.base.Preconditions;
+
+public class S3MinioDocker {
+
+ public static final DockerImageName DOCKER_IMAGE_NAME = DockerImageName.parse("minio/minio")
+ .withTag("RELEASE.2024-10-13T13-34-11Z");
+
+ public static final int MINIO_PORT = 9000;
+ public static final int MINIO_WEB_ADMIN_PORT = 9090;
+ public static final String MINIO_ROOT_USER = "minio";
+ public static final String MINIO_ROOT_PASSWORD = "minio123";
+
+ private final GenericContainer<?> container;
+
+ public S3MinioDocker() {
+ this.container = getContainer();
+ }
+
+ public S3MinioDocker(Network network) {
+ this.container = getContainer()
+ .withNetwork(network);
+ }
+
+ private GenericContainer<?> getContainer() {
+ return new GenericContainer<>(DOCKER_IMAGE_NAME)
+ .withExposedPorts(MINIO_PORT, MINIO_WEB_ADMIN_PORT)
+ .withEnv("MINIO_ROOT_USER", MINIO_ROOT_USER)
+ .withEnv("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD)
+ .withCommand("server", "--certs-dir", "/opt/minio/certs", "/data", "--console-address", ":" + MINIO_WEB_ADMIN_PORT)
+ .withClasspathResourceMapping("/minio/private.key",
+ "/opt/minio/certs/private.key",
+ BindMode.READ_ONLY)
+ .withClasspathResourceMapping("/minio/public.crt",
+ "/opt/minio/certs/public.crt",
+ BindMode.READ_ONLY)
+ .waitingFor(Wait.forLogMessage(".*MinIO Object Storage Server.*", 1)
+ .withStartupTimeout(Duration.ofMinutes(2)))
+ .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-minio-s3-test-" + UUID.randomUUID()));
+ }
+
+ public void start() {
+ if (!container.isRunning()) {
+ container.start();
+ setupMC();
+ }
+ }
+
+ public void stop() {
+ container.stop();
+ }
+
+ public AwsS3AuthConfiguration getAwsS3AuthConfiguration() {
+ Preconditions.checkArgument(container.isRunning(), "Container is not running");
+ return AwsS3AuthConfiguration.builder()
+ .endpoint(Throwing.supplier(() -> new URIBuilder()
+ .setScheme("https")
+ .setHost(container.getHost())
+ .setPort(container.getMappedPort(MINIO_PORT))
+ .build()).get())
+ .accessKeyId(MINIO_ROOT_USER)
+ .secretKey(MINIO_ROOT_PASSWORD)
+ .trustAll(true)
+ .build();
+ }
+
+ private void setupMC() {
+ Preconditions.checkArgument(container.isRunning(), "Container is not running");
+ Throwing.runnable(() -> container.execInContainer("mc", "alias", "set", "--insecure", "james", "https://localhost:9000", MINIO_ROOT_USER, MINIO_ROOT_PASSWORD)).run();
+ }
+
+ public void flushAll() {
+ // Remove all objects
+ Throwing.runnable(() -> container.execInContainer("mc", "--insecure", "rm", "--recursive", "--force", "--dangerous", "james/")).run();
+ // Remove all buckets
+ Throwing.runnable(() -> container.execInContainer("mc", "--insecure", "rb", "--force", "--dangerous", "james/")).run();
+ }
+}
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java
new file mode 100644
index 0000000000..12d95b3170
--- /dev/null
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioExtension.java
@@ -0,0 +1,59 @@
+/****************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one *
+ * or more contributor license agreements. See the NOTICE file *
+ * distributed with this work for additional information *
+ * regarding copyright ownership. The ASF licenses this file *
+ * to you under the Apache License, Version 2.0 (the *
+ * "License"); you may not use this file except in compliance *
+ * with the License. You may obtain a copy of the License at *
+ * *
+ * http://www.apache.org/licenses/LICENSE-2.0 *
+ * *
+ * Unless required by applicable law or agreed to in writing, *
+ * software distributed under the License is distributed on an *
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY *
+ * KIND, either express or implied. See the License for the *
+ * specific language governing permissions and limitations *
+ * under the License. *
+ ****************************************************************/
+
+package org.apache.james.blob.objectstorage.aws;
+
+import org.apache.james.GuiceModuleTestExtension;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.api.extension.ParameterContext;
+import org.junit.jupiter.api.extension.ParameterResolutionException;
+
+public class S3MinioExtension implements GuiceModuleTestExtension {
+
+ private static final S3MinioDocker s3MinioDocker = new S3MinioDocker();
+
+ @Override
+ public void beforeAll(ExtensionContext extensionContext) {
+ s3MinioDocker.start();
+ }
+
+ @Override
+ public void afterAll(ExtensionContext extensionContext) {
+ s3MinioDocker.stop();
+ }
+
+ @Override
+ public void afterEach(ExtensionContext extensionContext) {
+ s3MinioDocker.flushAll();
+ }
+
+ @Override
+ public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return parameterContext.getParameter().getType() == S3MinioDocker.class;
+ }
+
+ @Override
+ public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) throws ParameterResolutionException {
+ return s3MinioDocker;
+ }
+
+ public S3MinioDocker minioDocker() {
+ return s3MinioDocker;
+ }
+}
diff --git a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
index 6f372240f3..ad506127d2 100644
--- a/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
+++ b/server/blob/blob-s3/src/test/java/org/apache/james/blob/objectstorage/aws/S3MinioTest.java
@@ -22,15 +22,11 @@ package org.apache.james.blob.objectstorage.aws;
import static org.apache.james.blob.api.BlobStoreDAOFixture.SHORT_BYTEARRAY;
import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BLOB_ID;
import static org.apache.james.blob.api.BlobStoreDAOFixture.TEST_BUCKET_NAME;
-import static org.apache.james.blob.objectstorage.aws.DockerAwsS3Container.ACCESS_KEY_ID;
-import static org.apache.james.blob.objectstorage.aws.DockerAwsS3Container.SECRET_ACCESS_KEY;
import static org.apache.james.blob.objectstorage.aws.S3BlobStoreConfiguration.UPLOAD_RETRY_EXCEPTION_PREDICATE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-import java.net.URI;
import java.util.Optional;
-import java.util.UUID;
import java.util.concurrent.ExecutionException;
import org.apache.james.blob.api.BlobId;
@@ -40,50 +36,31 @@ import org.apache.james.blob.api.TestBlobId;
import org.apache.james.metrics.api.NoopGaugeRegistry;
import org.apache.james.metrics.tests.RecordingMetricFactory;
import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.junit.jupiter.Container;
-import org.testcontainers.junit.jupiter.Testcontainers;
+import org.junit.jupiter.api.extension.RegisterExtension;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.util.retry.Retry;
import software.amazon.awssdk.services.s3.model.S3Exception;
-@Testcontainers
public class S3MinioTest implements BlobStoreDAOContract {
- private static final String MINIO_IMAGE = "minio/minio";
- private static final String MINIO_TAG = "RELEASE.2024-10-13T13-34-11Z";
- private static final String MINIO_IMAGE_FULL = MINIO_IMAGE + ":" + MINIO_TAG;
- private static final int MINIO_PORT = 9000;
- private static S3BlobStoreDAO testee;
+ @RegisterExtension
+ static S3MinioExtension minoExtension = new S3MinioExtension();
+ private static S3BlobStoreDAO testee;
private static S3ClientFactory s3ClientFactory;
- @Container
- private static final GenericContainer<?> minioContainer = new GenericContainer<>(MINIO_IMAGE_FULL)
- .withExposedPorts(MINIO_PORT)
- .withEnv("MINIO_ROOT_USER", ACCESS_KEY_ID)
- .withEnv("MINIO_ROOT_PASSWORD", SECRET_ACCESS_KEY)
- .withCommand("server", "/data", "--console-address", ":9090")
- .withCreateContainerCmdModifier(createContainerCmd -> createContainerCmd.withName("james-minio-s3-test-" + UUID.randomUUID()));
-
-
@BeforeAll
static void setUp() {
- AwsS3AuthConfiguration authConfiguration = AwsS3AuthConfiguration.builder()
- .endpoint(URI.create(String.format("http://%s:%s/", minioContainer.getHost(), minioContainer.getMappedPort(MINIO_PORT))))
- .accessKeyId(ACCESS_KEY_ID)
- .secretKey(SECRET_ACCESS_KEY)
- .build();
+ AwsS3AuthConfiguration awsS3AuthConfiguration = minoExtension.minioDocker().getAwsS3AuthConfiguration();
S3BlobStoreConfiguration s3Configuration = S3BlobStoreConfiguration.builder()
- .authConfiguration(authConfiguration)
+ .authConfiguration(awsS3AuthConfiguration)
.region(DockerAwsS3Container.REGION)
.uploadRetrySpec(Optional.of(Retry.backoff(3, java.time.Duration.ofSeconds(1))
.filter(UPLOAD_RETRY_EXCEPTION_PREDICATE)))
@@ -98,11 +75,6 @@ public class S3MinioTest implements BlobStoreDAOContract {
s3ClientFactory.close();
}
- @AfterEach
- void tearDown() {
- testee.deleteAllBuckets().block();
- }
-
@BeforeEach
void beforeEach() throws Exception {
// Why? https://github.com/apache/james-project/pull/1981#issuecomment-2380396460
diff --git a/server/blob/blob-s3/src/test/resources/minio/private.key b/server/blob/blob-s3/src/test/resources/minio/private.key
new file mode 100644
index 0000000000..f829a5dd37
--- /dev/null
+++ b/server/blob/blob-s3/src/test/resources/minio/private.key
@@ -0,0 +1,5 @@
+-----BEGIN PRIVATE KEY-----
+MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgDhp6TEYrEFHMHusJ
+Xstem4lh0czPokiWEdAzzMXRXxGhRANCAAQ7jB4mm+Rieqr3lQfdc1xoPsgAltNP
+CvUYDZyu5903CkLiDuhBU6UWvQP5F19XFt95RkaYDPvc6ka0485NWFMb
+-----END PRIVATE KEY-----
diff --git a/server/blob/blob-s3/src/test/resources/minio/public.crt b/server/blob/blob-s3/src/test/resources/minio/public.crt
new file mode 100644
index 0000000000..5a6013846c
--- /dev/null
+++ b/server/blob/blob-s3/src/test/resources/minio/public.crt
@@ -0,0 +1,14 @@
+-----BEGIN CERTIFICATE-----
+MIICFjCCAbugAwIBAgIRAK8/sg1OIuWOQV2gWo9vDQUwCgYIKoZIzj0EAwIwTTEc
+MBoGA1UEChMTQ2VydGdlbiBEZXZlbG9wbWVudDEtMCsGA1UECwwkdHVuZ3R2QE1h
+Y0Jvb2stUHJvLmxvY2FsIChUcmFuIFR1bmcpMB4XDTI0MDkyNjAyNDcwNFoXDTI1
+MDkyNjAyNDcwNFowTTEcMBoGA1UEChMTQ2VydGdlbiBEZXZlbG9wbWVudDEtMCsG
+A1UECwwkdHVuZ3R2QE1hY0Jvb2stUHJvLmxvY2FsIChUcmFuIFR1bmcpMFkwEwYH
+KoZIzj0CAQYIKoZIzj0DAQcDQgAEO4weJpvkYnqq95UH3XNcaD7IAJbTTwr1GA2c
+rufdNwpC4g7oQVOlFr0D+RdfVxbfeUZGmAz73OpGtOPOTVhTG6N8MHowDgYDVR0P
+AQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8w
+HQYDVR0OBBYEFKIu0NT9bG853GA+o10GsbDOi9zMMCMGA1UdEQQcMBqCCWxvY2Fs
+aG9zdIINczNtaW5pby5sb2NhbDAKBggqhkjOPQQDAgNJADBGAiEAxLfNK4gPWw0G
+8QAv0QzARY0L5+8Lmpf5SwVzrjZ+zosCIQCA41tgccZ20+W1igansLkqnSTcY8Nw
+mtKwMTxUEwNRTw==
+-----END CERTIFICATE-----
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org