You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/08/22 16:28:35 UTC
[hadoop] branch trunk updated: HADOOP-16470. Make last AWS
credential provider in default auth chain
EC2ContainerCredentialsProviderWrapper.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 61b2df2 HADOOP-16470. Make last AWS credential provider in default auth chain EC2ContainerCredentialsProviderWrapper.
61b2df2 is described below
commit 61b2df23317767833b327e5f69dbe73e8d4f0fc1
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Thu Aug 22 17:23:58 2019 +0100
HADOOP-16470. Make last AWS credential provider in default auth chain EC2ContainerCredentialsProviderWrapper.
Contributed by Steve Loughran.
Contains HADOOP-16471. Restore (documented) fs.s3a.SharedInstanceProfileCredentialsProvider.
Change-Id: I06b99b57459cac80bf743c5c54f04e59bb54c2f8
---
.../src/main/resources/core-default.xml | 4 +-
.../fs/s3a/SharedInstanceCredentialProvider.java | 44 ++++++++++++++++++++++
.../s3a/auth/IAMInstanceCredentialsProvider.java | 34 ++++++++++-------
.../src/site/markdown/tools/hadoop-aws/index.md | 4 +-
4 files changed, 68 insertions(+), 18 deletions(-)
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 05bba0e..2b78ede 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1092,8 +1092,8 @@
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- * com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
- of instance profile credentials if running in an EC2 VM.
+ * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
+ IAM credentials of any EC2 VM or AWS container in which the process is running.
</description>
</property>
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
new file mode 100644
index 0000000..5eba675
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceCredentialProvider.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider;
+import org.apache.hadoop.fs.s3a.auth.NoAwsCredentialsException;
+
+/**
+ * This credential provider has jittered between existing and non-existing,
+ * but it turns up in documentation enough that it has been restored.
+ * It extends {@link IAMInstanceCredentialsProvider} to pick up its
+ * bindings, which are currently to use the
+ * {@code EC2ContainerCredentialsProviderWrapper} class for IAM and container
+ * authentication.
+ * <p>
+ * When it fails to authenticate, it raises a
+ * {@link NoAwsCredentialsException} which can be recognized by retry handlers
+ * as a non-recoverable failure.
+ * <p>
+ * It is implicitly public; marked evolving as we can change its semantics.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class SharedInstanceCredentialProvider extends
+ IAMInstanceCredentialsProvider {
+}
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
index 7ff4510..1bb30ed 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/auth/IAMInstanceCredentialsProvider.java
@@ -24,38 +24,44 @@ import java.io.IOException;
import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
/**
- * This is going to be an IAM credential provider which performs
- * async refresh for lower-latency on IO calls.
- * Initially it does not do this, simply shares the single IAM instance
- * across all instances. This makes it less expensive to declare.
- *
+ * This is an IAM credential provider which wraps
+ * an {@code EC2ContainerCredentialsProviderWrapper}
+ * to provide credentials when the S3A connector is instantiated on AWS EC2
+ * or the AWS container services.
+ * <p>
+ * When it fails to authenticate, it raises a
+ * {@link NoAwsCredentialsException} which can be recognized by retry handlers
+ * as a non-recoverable failure.
+ * <p>
+ * It is implicitly public; marked evolving as we can change its semantics.
*/
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class IAMInstanceCredentialsProvider
implements AWSCredentialsProvider, Closeable {
- private static final InstanceProfileCredentialsProvider INSTANCE =
- InstanceProfileCredentialsProvider.getInstance();
+ private final AWSCredentialsProvider provider =
+ new EC2ContainerCredentialsProviderWrapper();
public IAMInstanceCredentialsProvider() {
}
/**
* Ask for the credentials.
- * as it invariably means "you aren't running on EC2"
+ * Failure invariably means "you aren't running in an EC2 VM or AWS container".
* @return the credentials
+ * @throws NoAwsCredentialsException on auth failure to indicate non-recoverable.
*/
@Override
public AWSCredentials getCredentials() {
try {
- return INSTANCE.getCredentials();
+ return provider.getCredentials();
} catch (AmazonClientException e) {
throw new NoAwsCredentialsException("IAMInstanceCredentialsProvider",
e.getMessage(),
@@ -65,11 +71,11 @@ public class IAMInstanceCredentialsProvider
@Override
public void refresh() {
- INSTANCE.refresh();
+ provider.refresh();
}
@Override
public void close() throws IOException {
- // until async, no-op.
+ // no-op.
}
}
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 704e49b..7b6eb83 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -539,8 +539,8 @@ This means that the default S3A authentication chain can be defined as
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- * com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
- of instance profile credentials if running in an EC2 VM.
+ * org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider: picks up
+ IAM credentials of any EC2 VM or AWS container in which the process is running.
</description>
</property>
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org