You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2018/01/29 22:59:14 UTC
[beam] branch master updated: [BEAM-3550] Add --awsServiceEndpoint
option and use with S3 filesystem.
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new bc3bc68 [BEAM-3550] Add --awsServiceEndpoint option and use with S3 filesystem.
bc3bc68 is described below
commit bc3bc68974e4e16fc4f30247695041e22a9d4a6a
Author: John MacMillan <jo...@ca.ibm.com>
AuthorDate: Mon Jan 29 15:24:59 2018 -0500
[BEAM-3550] Add --awsServiceEndpoint option and use with S3 filesystem.
If the option is null or empty, the default endpoint will be used.
---
.../org/apache/beam/sdk/io/aws/options/AwsOptions.java | 7 +++++++
.../java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java | 15 ++++++++++-----
2 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
index 8ed68e5..ce55792 100644
--- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
+++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/options/AwsOptions.java
@@ -40,6 +40,13 @@ public interface AwsOptions extends PipelineOptions {
void setAwsRegion(String value);
/**
+ * The AWS service endpoint used by the AWS client.
+ */
+ @Description("AWS service endpoint used by the AWS client")
+ String getAwsServiceEndpoint();
+ void setAwsServiceEndpoint(String value);
+
+ /**
* The credential instance that should be used to authenticate against AWS services. Refer to
* {@link DefaultAWSCredentialsProviderChain} Javadoc for usage help.
*/
diff --git a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
index 11c3418..5adf42a 100644
--- a/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
+++ b/sdks/java/io/amazon-web-services/src/main/java/org/apache/beam/sdk/io/aws/s3/S3FileSystem.java
@@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import com.amazonaws.AmazonClientException;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
@@ -106,11 +107,15 @@ class S3FileSystem extends FileSystem<S3ResourceId> {
+ "was not specified. If you don't plan to use S3, then ignore this message.");
}
- amazonS3 =
- AmazonS3ClientBuilder.standard()
- .withCredentials(options.getAwsCredentialsProvider())
- .withRegion(options.getAwsRegion())
- .build();
+ AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard()
+ .withCredentials(options.getAwsCredentialsProvider());
+ if (Strings.isNullOrEmpty(options.getAwsServiceEndpoint())) {
+ builder = builder.withRegion(options.getAwsRegion());
+ } else {
+ builder = builder.withEndpointConfiguration(new EndpointConfiguration(
+ options.getAwsServiceEndpoint(), options.getAwsRegion()));
+ }
+ amazonS3 = builder.build();
this.storageClass = checkNotNull(options.getS3StorageClass(), "storageClass");
--
To stop receiving notification emails like this one, please contact
lcwik@apache.org.