You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by li...@apache.org on 2017/04/12 22:53:06 UTC
[41/50] hadoop git commit: HADOOP-14248. Retire
SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang
Liu.
HADOOP-14248. Retire SharedInstanceProfileCredentialsProvider in trunk. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8305e6d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8305e6d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8305e6d
Branch: refs/heads/HADOOP-13345
Commit: b8305e6d06cbb9d44c728da14506d5cf825c12e5
Parents: a16ab2b
Author: Chris Nauroth <cn...@apache.org>
Authored: Wed Apr 12 10:02:13 2017 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Wed Apr 12 10:02:13 2017 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 9 +--
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 8 +--
...haredInstanceProfileCredentialsProvider.java | 67 --------------------
.../src/site/markdown/tools/hadoop-aws/index.md | 33 +++-------
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 4 +-
5 files changed, 13 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 03e4996..4f37c65 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -955,13 +955,8 @@
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
- a shared instance of
- com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
- SDK, which supports use of instance profile credentials if running
- in an EC2 VM. Using this shared instance potentially reduces load
- on the EC2 instance metadata service for multi-threaded
- applications.
+ 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+ of instance profile credentials if running in an EC2 VM.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6a11699..5ff9321 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -339,15 +339,9 @@ public final class S3AUtils {
credentials.add(new BasicAWSCredentialsProvider(
creds.getUser(), creds.getPassword()));
credentials.add(new EnvironmentVariableCredentialsProvider());
- credentials.add(
- SharedInstanceProfileCredentialsProvider.getInstance());
+ credentials.add(InstanceProfileCredentialsProvider.getInstance());
} else {
for (Class<?> aClass : awsClasses) {
- if (aClass == InstanceProfileCredentialsProvider.class) {
- LOG.debug("Found {}, but will use {} instead.", aClass.getName(),
- SharedInstanceProfileCredentialsProvider.class.getName());
- aClass = SharedInstanceProfileCredentialsProvider.class;
- }
credentials.add(createAWSCredentialProvider(conf, aClass));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
deleted file mode 100644
index cbc0787..0000000
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/SharedInstanceProfileCredentialsProvider.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.s3a;
-
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * A subclass of {@link InstanceProfileCredentialsProvider} that enforces
- * instantiation of only a single instance.
- * This credential provider calls the EC2 instance metadata service to obtain
- * credentials. For highly multi-threaded applications, it's possible that
- * multiple instances call the service simultaneously and overwhelm it with
- * load. The service handles this by throttling the client with an HTTP 429
- * response or forcibly terminating the connection. Forcing use of a single
- * instance reduces load on the metadata service by allowing all threads to
- * share the credentials. The base class is thread-safe, and there is nothing
- * that varies in the credentials across different instances of
- * {@link S3AFileSystem} connecting to different buckets, so sharing a singleton
- * instance is safe.
- *
- * As of AWS SDK 1.11.39, the SDK code internally enforces a singleton. After
- * Hadoop upgrades to that version or higher, it's likely that we can remove
- * this class.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Stable
-public final class SharedInstanceProfileCredentialsProvider
- extends InstanceProfileCredentialsProvider {
-
- private static final SharedInstanceProfileCredentialsProvider INSTANCE =
- new SharedInstanceProfileCredentialsProvider();
-
- /**
- * Returns the singleton instance.
- *
- * @return singleton instance
- */
- public static SharedInstanceProfileCredentialsProvider getInstance() {
- return INSTANCE;
- }
-
- /**
- * Default constructor, defined explicitly as private to enforce singleton.
- */
- private SharedInstanceProfileCredentialsProvider() {
- super();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 18c0ceb..a3a0bb1 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -328,13 +328,8 @@ of `com.amazonaws.auth.AWSCredentialsProvider` may also be used.
configuration of AWS access key ID and secret access key in
environment variables named AWS_ACCESS_KEY_ID and
AWS_SECRET_ACCESS_KEY, as documented in the AWS SDK.
- 3. org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider:
- a shared instance of
- com.amazonaws.auth.InstanceProfileCredentialsProvider from the AWS
- SDK, which supports use of instance profile credentials if running
- in an EC2 VM. Using this shared instance potentially reduces load
- on the EC2 instance metadata service for multi-threaded
- applications.
+ 3. com.amazonaws.auth.InstanceProfileCredentialsProvider: supports use
+ of instance profile credentials if running in an EC2 VM.
</description>
</property>
@@ -407,13 +402,12 @@ AWS Credential Providers are classes which can be used by the Amazon AWS SDK to
obtain an AWS login from a different source in the system, including environment
variables, JVM properties and configuration files.
-There are four AWS Credential Providers inside the `hadoop-aws` JAR:
+There are three AWS Credential Providers inside the `hadoop-aws` JAR:
| classname | description |
|-----------|-------------|
| `org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider`| Session Credentials |
| `org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider`| Simple name/secret credentials |
-| `org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider`| Shared instance of EC2 Metadata Credentials, which can reduce load on the EC2 instance metadata service. (See below.) |
| `org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider`| Anonymous Login |
There are also many in the Amazon SDKs, in particular two which are automatically
@@ -425,24 +419,13 @@ set up in the authentication chain:
| `com.amazonaws.auth.EnvironmentVariableCredentialsProvider`| AWS Environment Variables |
-*EC2 Metadata Credentials with `SharedInstanceProfileCredentialsProvider`*
+*EC2 Metadata Credentials with `InstanceProfileCredentialsProvider`*
Applications running in EC2 may associate an IAM role with the VM and query the
[EC2 Instance Metadata Service](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html)
for credentials to access S3. Within the AWS SDK, this functionality is
-provided by `InstanceProfileCredentialsProvider`. Heavily multi-threaded
-applications may trigger a high volume of calls to the instance metadata service
-and trigger throttling: either an HTTP 429 response or a forcible close of the
-connection.
-
-To mitigate against this problem, `hadoop-aws` ships with a variant of
-`InstanceProfileCredentialsProvider` called
-`SharedInstanceProfileCredentialsProvider`. Using this ensures that all
-instances of S3A reuse the same instance profile credentials instead of issuing
-a large volume of redundant metadata service calls. If
-`fs.s3a.aws.credentials.provider` refers to
-`com.amazonaws.auth.InstanceProfileCredentialsProvider`, S3A automatically uses
-`org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider` instead.
+provided by `InstanceProfileCredentialsProvider`, which internally enforces a
+singleton instance in order to prevent throttling problem.
*Session Credentials with `TemporaryAWSCredentialsProvider`*
@@ -542,7 +525,7 @@ This means that the default S3A authentication chain can be defined as
<value>
org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider,
com.amazonaws.auth.EnvironmentVariableCredentialsProvider,
- org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider
+ com.amazonaws.auth.InstanceProfileCredentialsProvider
</value>
</property>
@@ -929,7 +912,7 @@ role information available when deployed in Amazon EC2.
```xml
<property>
<name>fs.s3a.aws.credentials.provider</name>
- <value>org.apache.hadoop.fs.s3a.SharedInstanceProfileCredentialsProvider</value>
+ <value>com.amazonaws.auth.InstanceProfileCredentialsProvider</value>
</property>
```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8305e6d/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
index 33740c8..82a8b84 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AAWSCredentialsProvider.java
@@ -114,7 +114,7 @@ public class TestS3AAWSCredentialsProvider {
Arrays.asList(
BasicAWSCredentialsProvider.class,
EnvironmentVariableCredentialsProvider.class,
- SharedInstanceProfileCredentialsProvider.class);
+ InstanceProfileCredentialsProvider.class);
assertCredentialProviders(expectedClasses, list1);
assertCredentialProviders(expectedClasses, list2);
assertSameInstanceProfileCredentialsProvider(list1.getProviders().get(2),
@@ -128,7 +128,7 @@ public class TestS3AAWSCredentialsProvider {
List<Class<? extends AWSCredentialsProvider>> expectedClasses =
Arrays.asList(
EnvironmentVariableCredentialsProvider.class,
- SharedInstanceProfileCredentialsProvider.class,
+ InstanceProfileCredentialsProvider.class,
AnonymousAWSCredentialsProvider.class);
conf.set(AWS_CREDENTIALS_PROVIDER, buildClassListString(expectedClasses));
AWSCredentialProviderList list1 = S3AUtils.createAWSCredentialProviderSet(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org