You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@james.apache.org by rc...@apache.org on 2021/05/21 01:51:06 UTC
[james-project] 02/02: JAMES-3028 Allow setting up S3 HTTP
concurrency at the Netty level
This is an automated email from the ASF dual-hosted git repository.
rcordier pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/james-project.git
commit 5e52bc456ca12060d4342600466f1051605c3fd0
Author: Benoit Tellier <bt...@linagora.com>
AuthorDate: Tue May 18 08:00:15 2021 +0700
JAMES-3028 Allow setting up S3 HTTP concurrency at the Netty level
---
.../pages/distributed/configure/blobstore.adoc | 3 +++
.../aws/S3BlobStoreConfiguration.java | 24 +++++++++++++++++++---
.../blob/objectstorage/aws/S3BlobStoreDAO.java | 2 +-
.../S3BlobStoreConfigurationReader.java | 3 +++
src/site/xdoc/server/config-blobstore.xml | 3 +++
5 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/docs/modules/servers/pages/distributed/configure/blobstore.adoc b/docs/modules/servers/pages/distributed/configure/blobstore.adoc
index c256b13..5611040 100644
--- a/docs/modules/servers/pages/distributed/configure/blobstore.adoc
+++ b/docs/modules/servers/pages/distributed/configure/blobstore.adoc
@@ -109,6 +109,9 @@ Maximum size of stored objects expressed in bytes.
| objectstorage.s3.secretKey
| https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys[S3 access key secret]
+
+| objectstorage.s3.http.concurrency
+| Allow setting the number of concurrent HTTP requests allowed by the Netty driver.
|===
==== Buckets Configuration
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java
index d0a1b7a..a456d4a 100644
--- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreConfiguration.java
@@ -51,6 +51,7 @@ public class S3BlobStoreConfiguration {
private Optional<BucketName> defaultBucketName;
private Optional<String> bucketPrefix;
+ private Optional<Integer> httpConcurrency;
private Region region;
public ReadyToBuild(AwsS3AuthConfiguration specificAuthConfiguration, Region region) {
@@ -58,6 +59,7 @@ public class S3BlobStoreConfiguration {
this.region = region;
this.defaultBucketName = Optional.empty();
this.bucketPrefix = Optional.empty();
+ this.httpConcurrency = Optional.empty();
}
public ReadyToBuild defaultBucketName(Optional<BucketName> defaultBucketName) {
@@ -80,27 +82,37 @@ public class S3BlobStoreConfiguration {
return this;
}
+ public ReadyToBuild httpConcurrency(Optional<Integer> httpConcurrency) {
+ this.httpConcurrency = httpConcurrency;
+ return this;
+ }
+
public S3BlobStoreConfiguration build() {
- return new S3BlobStoreConfiguration(bucketPrefix, defaultBucketName, region, specificAuthConfiguration);
+ return new S3BlobStoreConfiguration(bucketPrefix, defaultBucketName, region, specificAuthConfiguration, httpConcurrency.orElse(DEFAULT_HTTP_CONCURRENCY));
}
}
}
+ public static int DEFAULT_HTTP_CONCURRENCY = 100;
+
private final Region region;
private final AwsS3AuthConfiguration specificAuthConfiguration;
private final Optional<BucketName> namespace;
private final Optional<String> bucketPrefix;
+ private final int httpConcurrency;
@VisibleForTesting
S3BlobStoreConfiguration(Optional<String> bucketPrefix,
Optional<BucketName> namespace,
Region region,
- AwsS3AuthConfiguration specificAuthConfiguration) {
+ AwsS3AuthConfiguration specificAuthConfiguration,
+ int httpConcurrency) {
this.bucketPrefix = bucketPrefix;
this.namespace = namespace;
this.region = region;
this.specificAuthConfiguration = specificAuthConfiguration;
+ this.httpConcurrency = httpConcurrency;
}
public Optional<BucketName> getNamespace() {
@@ -115,6 +127,10 @@ public class S3BlobStoreConfiguration {
return bucketPrefix;
}
+ public int getHttpConcurrency() {
+ return httpConcurrency;
+ }
+
public Region getRegion() {
return region;
}
@@ -127,6 +143,7 @@ public class S3BlobStoreConfiguration {
return Objects.equals(this.namespace, that.namespace)
&& Objects.equals(this.bucketPrefix, that.bucketPrefix)
&& Objects.equals(this.region, that.region)
+ && Objects.equals(this.httpConcurrency, that.httpConcurrency)
&& Objects.equals(this.specificAuthConfiguration, that.specificAuthConfiguration);
}
return false;
@@ -134,13 +151,14 @@ public class S3BlobStoreConfiguration {
@Override
public final int hashCode() {
- return Objects.hash(namespace, bucketPrefix, specificAuthConfiguration);
+ return Objects.hash(namespace, bucketPrefix, httpConcurrency, specificAuthConfiguration);
}
@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("namespace", namespace)
+ .add("httpConcurrency", httpConcurrency)
.add("bucketPrefix", bucketPrefix)
.add("region", region)
.add("specificAuthConfiguration", specificAuthConfiguration)
diff --git a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
index d228aee..0a12fea 100644
--- a/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
+++ b/server/blob/blob-s3/src/main/java/org/apache/james/blob/objectstorage/aws/S3BlobStoreDAO.java
@@ -97,7 +97,7 @@ public class S3BlobStoreDAO implements BlobStoreDAO, Startable, Closeable {
.credentialsProvider(StaticCredentialsProvider.create(
AwsBasicCredentials.create(authConfiguration.getAccessKeyId(), authConfiguration.getSecretKey())))
.httpClientBuilder(NettyNioAsyncHttpClient.builder()
- .maxConcurrency(100)
+ .maxConcurrency(configuration.getHttpConcurrency())
.maxPendingConnectionAcquires(10_000))
.endpointOverride(authConfiguration.getEndpoint())
.region(configuration.getRegion().asAws())
diff --git a/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java b/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java
index 418c7a7..44b74f6 100644
--- a/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java
+++ b/server/container/guice/blob/s3/src/main/java/org/apache/james/modules/objectstorage/S3BlobStoreConfigurationReader.java
@@ -33,8 +33,10 @@ public class S3BlobStoreConfigurationReader {
private static final String OBJECTSTORAGE_NAMESPACE = "objectstorage.namespace";
private static final String OBJECTSTORAGE_BUCKET_PREFIX = "objectstorage.bucketPrefix";
private static final String OBJECTSTORAGE_S3_REGION = "objectstorage.s3.region";
+ private static final String OBJECTSTORAGE_S3_HTTP_CONCURRENCY = "objectstorage.s3.http.concurrency";
public static S3BlobStoreConfiguration from(Configuration configuration) throws ConfigurationException {
+ Optional<Integer> httpConcurrency = Optional.ofNullable(configuration.getInteger(OBJECTSTORAGE_S3_HTTP_CONCURRENCY, null));
Optional<String> namespace = Optional.ofNullable(configuration.getString(OBJECTSTORAGE_NAMESPACE, null));
Optional<String> bucketPrefix = Optional.ofNullable(configuration.getString(OBJECTSTORAGE_BUCKET_PREFIX, null));
Region region = Optional.ofNullable(configuration.getString(OBJECTSTORAGE_S3_REGION, null))
@@ -46,6 +48,7 @@ public class S3BlobStoreConfigurationReader {
.region(region)
.defaultBucketName(namespace.map(BucketName::of))
.bucketPrefix(bucketPrefix)
+ .httpConcurrency(httpConcurrency)
.build();
}
diff --git a/src/site/xdoc/server/config-blobstore.xml b/src/site/xdoc/server/config-blobstore.xml
index 15ae7b2..513f8ca 100644
--- a/src/site/xdoc/server/config-blobstore.xml
+++ b/src/site/xdoc/server/config-blobstore.xml
@@ -149,6 +149,9 @@ generate salt with : openssl rand -hex 16
<dt><strong>objectstorage.s3.secretKey</strong></dt>
<dd><a href="https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys">S3 access key secret</a></dd>
+
+ <dt><strong>objectstorage.s3.http.concurrency</strong></dt>
+ <dd>Allow setting the number of concurrent HTTP requests allowed by the Netty driver.</dd>
</dl>
</subsection>
</subsection>
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@james.apache.org
For additional commands, e-mail: notifications-help@james.apache.org