You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2017/04/12 17:11:05 UTC
hadoop git commit: HADOOP-14301. Deprecate
SharedInstanceProfileCredentialsProvider in branch-2. Contributed by
Mingliang Liu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 607295d21 -> 1bfb38229
HADOOP-14301. Deprecate SharedInstanceProfileCredentialsProvider in branch-2. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1bfb3822
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1bfb3822
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1bfb3822
Branch: refs/heads/branch-2
Commit: 1bfb382296ab6f980341c926801b0eec57a68b87
Parents: 607295d
Author: Chris Nauroth <cn...@apache.org>
Authored: Wed Apr 12 10:07:00 2017 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Wed Apr 12 10:07:00 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 13 ++++----
...haredInstanceProfileCredentialsProvider.java | 9 +++--
.../src/site/markdown/tools/hadoop-aws/index.md | 35 ++++++++++----------
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 14 ++++++--
4 files changed, 41 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bfb3822/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6a11699..b652b3b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -339,14 +339,15 @@ public final class S3AUtils {
credentials.add(new BasicAWSCredentialsProvider(
creds.getUser(), creds.getPassword()));
credentials.add(new EnvironmentVariableCredentialsProvider());
- credentials.add(
- SharedInstanceProfileCredentialsProvider.getInstance());
+ credentials.add(InstanceProfileCredentialsProvider.getInstance());
} else {
for (Class<?> aClass : awsClasses) {
- if (aClass == InstanceProfileCredentialsProvider.class) {
- LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
- SharedInstanceProfileCredentialsProvider.class.getName());
- aClass = SharedInstanceProfileCredentialsProvider.class;
+ if (aClass == SharedInstanceProfileCredentialsProvider.class) {
+ LOG.warn("{} is deprecated and will be removed in future. " +
+ "Fall back to {} automatically.",
+ aClass.getName(),
+ InstanceProfileCredentialsProvider.class.getName());
+ aClass = InstanceProfileCredentialsProvider.class;
}
credentials.add(createAWSCredentialProvider(conf, aClass));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bfb3822/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
index cbc0787..d30444b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
@@ -37,12 +37,15 @@ import org.apache.hadoop.classification.InterfaceStability;
* {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
* instance is safe.
*
- * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After
- * Hadoop upgrades to that version or higher, it's likely that we can remove
- * this class.
+ * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. Hadoop
+ * has upgraded its dependency to 1.11.39+ so this class is deprecated. In
+ * next major version, this will be removed.
+ *
+ * @deprecated Please use {@link InstanceProfileCredentialsProvider} instead.
*/
@InterfaceAudience.Private
@InterfaceStability.Stable
+@Deprecated
public final class SharedInstanceProfileCredentialsProvider
extends InstanceProfileCredentialsProvider {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bfb3822/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 07cc903..4174141 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -357,13 +357,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
- a shared instance of
- com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
- SDK, which supports use of instance profile credentials if running
- in an EC2 VM. Using this shared instance potentially reduces load
- on the EC2 instance metadata service for multi-threaded
- applications.
+ 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+ of instance profile credentials if running in an EC2 VM.
</description>
</property>
@@ -459,19 +454,23 @@ set up in the authentication chain:
Applications running in EC2 may associate an IAM role with the VM and query the
[EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
for credentials to access S3. Within the AWS SDK, this functionality is
-provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded
-applications may trigger a high volume of calls to the instance metadata service
-and trigger throttling: either an HTTP 429 response or a forcible close of the
-connection.
+provided by `InstanceProfileCredentialsProvider`. In previous AWS SDK versions
+earlier than 1.11.39, heavily multi-threaded applications may trigger a high
+volume of calls to the instance metadata service and trigger throttling: either
+an HTTP 429 response or a forcible close of the connection.
-To mitigate against this problem, `hadoop-aws` ships with a variant of
+To mitigate against that problem, `hadoop-aws` ships with a variant of
`InstanceProfileCredentialsProvider` called
`SharedInstanceProfileCredentialsProvider`. Using this ensures that all
instances of S3A reuse the same instance profile credentials instead of issuing
-a large volume of redundant metadata service calls. If
-`fs.s3a.aws.credentials.provider` refers to
-`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
-`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
+a large volume of redundant metadata service calls.
+
+As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. Hadoop has
+upgraded its dependency so that this class is deprecated. In next major
+version, this will be removed. If `fs.s3a.aws.credentials.provider` refers to
+`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`, S3A
+automatically uses `com.amazonaws.auth.InstanceProfileCredentialsProvider`
+instead, along with a warning message.
*Session Credentials with `TemporaryAWSCredentialsProvider`*
@@ -571,7 +570,7 @@ This means that the default S3A authentication chain can be defined as
<value>
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
- org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+ com.amazonaws.auth.InstanceProfileCredentialsProvider
</value>
</property>
@@ -958,7 +957,7 @@ role information available when deployed in Amazon EC2.
```xml
<property>
<name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
+ <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value>
</property>
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1bfb3822/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 33740c8..cfa6dd4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider {
Arrays.asList(
BasicAWSCredentialsProvider.class,
EnvironmentVariableCredentialsProvider.class,
- SharedInstanceProfileCredentialsProvider.class);
+ InstanceProfileCredentialsProvider.class);
assertCredentialProviders(expectedClasses, list1);
assertCredentialProviders(expectedClasses, list2);
assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
@@ -125,12 +125,20 @@ public class TestS3AAWSCredentialsProvider {
public void testConfiguredChain() throws Exception {
URI uri1 = new URI("s3a://bucket1"), uri2 = new URI("s3a://bucket2");
Configuration conf = new Configuration();
- List<Class<? extends AWSCredentialsProvider>> expectedClasses =
+ List<Class<? extends AWSCredentialsProvider>> originalClasses =
Arrays.asList(
EnvironmentVariableCredentialsProvider.class,
SharedInstanceProfileCredentialsProvider.class,
AnonymousAWSCredentialsProvider.class);
- conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
+ conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(originalClasses));
+
+ // SharedInstanceProfileCredentialsProvider is deprecated and should have
+ // been replaced with InstanceProfileCredentialsProvider automatically
+ List<Class<? extends AWSCredentialsProvider>> expectedClasses =
+ Arrays.asList(
+ EnvironmentVariableCredentialsProvider.class,
+ InstanceProfileCredentialsProvider.class,
+ AnonymousAWSCredentialsProvider.class);
AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(
uri1, conf);
AWSCredentialProviderList list2 = S3AUtils.createAWSCredentialProviderSet(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org