You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by gi...@apache.org on 2019/01/31 03:32:07 UTC
[incubator-druid] branch master updated: Support protocol
configuration for S3 (#6954)
This is an automated email from the ASF dual-hosted git repository.
gian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new d4fbbb8 Support protocol configuration for S3 (#6954)
d4fbbb8 is described below
commit d4fbbb8deb05bb6aaabe1aeb9e654c692028e4ff
Author: Jihoon Son <ji...@apache.org>
AuthorDate: Wed Jan 30 19:32:00 2019 -0800
Support protocol configuration for S3 (#6954)
* Support protocol configuration for S3
* Add doc
---
.../apache/druid/common/aws/AWSClientConfig.java | 8 ++++
docs/content/development/extensions-core/s3.md | 1 +
.../druid/storage/s3/S3StorageDruidModule.java | 47 +++++++++++++++++++---
3 files changed, 51 insertions(+), 5 deletions(-)
diff --git a/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java b/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java
index 198bae4..5a0a8b0 100644
--- a/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java
+++ b/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java
@@ -25,6 +25,9 @@ import com.fasterxml.jackson.annotation.JsonProperty;
public class AWSClientConfig
{
@JsonProperty
+ private String protocol = "https"; // The default of aws-java-sdk
+
+ @JsonProperty
private boolean disableChunkedEncoding = S3ClientOptions.DEFAULT_CHUNKED_ENCODING_DISABLED;
@JsonProperty
@@ -33,6 +36,11 @@ public class AWSClientConfig
@JsonProperty
protected boolean forceGlobalBucketAccessEnabled = S3ClientOptions.DEFAULT_FORCE_GLOBAL_BUCKET_ACCESS_ENABLED;
+ public String getProtocol()
+ {
+ return protocol;
+ }
+
public boolean isDisableChunkedEncoding()
{
return disableChunkedEncoding;
diff --git a/docs/content/development/extensions-core/s3.md b/docs/content/development/extensions-core/s3.md
index c1ad94f..c7ca62a 100644
--- a/docs/content/development/extensions-core/s3.md
+++ b/docs/content/development/extensions-core/s3.md
@@ -48,6 +48,7 @@ As an example, to set the region to 'us-east-1' through system properties:
|`druid.storage.sse.type`|Server-side encryption type. Should be one of `s3`, `kms`, and `custom`. See the below [Server-side encryption section](#server-side-encryption) for more details.|None|
|`druid.storage.sse.kms.keyId`|AWS KMS key ID. Can be empty if `druid.storage.sse.type` is `kms`.|None|
|`druid.storage.sse.custom.base64EncodedKey`|Base64-encoded key. Should be specified if `druid.storage.sse.type` is `custom`.|None|
+|`druid.s3.protocol`|Communication protocol type to use when sending requests to AWS. `http` or `https` can be used.|`https`|
|`druid.s3.disableChunkedEncoding`|Disables chunked encoding. See [AWS document](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#disableChunkedEncoding--) for details.|false|
|`druid.s3.enablePathStyleAccess`|Enables path style access. See [AWS document](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#enablePathStyleAccess--) for details.|false|
|`druid.s3.forceGlobalBucketAccessEnabled`|Enables global bucket access. See [AWS document](https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Builder.html#setForceGlobalBucketAccessEnabled-java.lang.Boolean-) for details.|false|
diff --git a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
index 445e906..d857363 100644
--- a/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
+++ b/extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3StorageDruidModule.java
@@ -21,6 +21,7 @@ package org.apache.druid.storage.s3;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.ClientConfigurationFactory;
+import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.services.s3.AmazonS3Client;
@@ -41,7 +42,11 @@ import org.apache.druid.guice.Binders;
import org.apache.druid.guice.JsonConfigProvider;
import org.apache.druid.guice.LazySingleton;
import org.apache.druid.initialization.DruidModule;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.logger.Logger;
+import javax.annotation.Nullable;
+import java.net.URI;
import java.util.List;
/**
@@ -50,6 +55,8 @@ public class S3StorageDruidModule implements DruidModule
{
public static final String SCHEME = "s3_zip";
+ private static final Logger log = new Logger(S3StorageDruidModule.class);
+
@Override
public List<? extends Module> getJacksonModules()
{
@@ -122,10 +129,11 @@ public class S3StorageDruidModule implements DruidModule
)
{
final ClientConfiguration configuration = new ClientConfigurationFactory().getConfig();
+ final Protocol protocol = determineProtocol(clientConfig, endpointConfig);
final AmazonS3ClientBuilder builder = AmazonS3Client
.builder()
.withCredentials(provider)
- .withClientConfiguration(setProxyConfig(configuration, proxyConfig))
+ .withClientConfiguration(setProxyConfig(configuration, proxyConfig).withProtocol(protocol))
.withChunkedEncodingDisabled(clientConfig.isDisableChunkedEncoding())
.withPathStyleAccessEnabled(clientConfig.isEnablePathStyleAccess())
.withForceGlobalBucketAccessEnabled(clientConfig.isForceGlobalBucketAccessEnabled());
@@ -136,10 +144,7 @@ public class S3StorageDruidModule implements DruidModule
);
}
- return new ServerSideEncryptingAmazonS3(
- builder.build(),
- storageConfig.getServerSideEncryption()
- );
+ return new ServerSideEncryptingAmazonS3(builder.build(), storageConfig.getServerSideEncryption());
}
private static ClientConfiguration setProxyConfig(ClientConfiguration conf, AWSProxyConfig proxyConfig)
@@ -158,4 +163,36 @@ public class S3StorageDruidModule implements DruidModule
}
return conf;
}
+
+ @Nullable
+ private static Protocol parseProtocol(@Nullable String protocol)
+ {
+ if (protocol == null) {
+ return null;
+ }
+
+ if (protocol.equalsIgnoreCase("http")) {
+ return Protocol.HTTP;
+ } else if (protocol.equalsIgnoreCase("https")) {
+ return Protocol.HTTPS;
+ } else {
+ throw new IAE("Unknown protocol[%s]", protocol);
+ }
+ }
+
+ private static Protocol determineProtocol(AWSClientConfig clientConfig, AWSEndpointConfig endpointConfig)
+ {
+ final Protocol protocolFromClientConfig = parseProtocol(clientConfig.getProtocol());
+ final String endpointUrl = endpointConfig.getUrl();
+ if (StringUtils.isNotEmpty(endpointUrl)) {
+ final URI uri = URI.create(endpointUrl);
+ final Protocol protocol = parseProtocol(uri.getScheme());
+ if (protocol != null && (protocol != protocolFromClientConfig)) {
+ log.warn("[%s] protocol will be used for endpoint [%s]", protocol, endpointUrl);
+ }
+ return protocol;
+ } else {
+ return protocolFromClientConfig;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org