You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/03/13 00:04:31 UTC
[12/49] hadoop git commit: HADOOP-11670. Regression: s3a auth setup
broken. (Adam Budde via stevel)
HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64443490
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64443490
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64443490
Branch: refs/heads/YARN-2928
Commit: 64443490d7f189e8e42d284615f3814ef751395a
Parents: 608ebd5
Author: Steve Loughran <st...@apache.org>
Authored: Sun Mar 8 11:20:42 2015 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Mar 8 11:22:16 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../org/apache/hadoop/fs/s3a/Constants.java | 6 +++++-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 20 ++++++++++++++------
.../src/site/markdown/tools/hadoop-aws/index.md | 10 +++++-----
4 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64443490/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 14cd75a..16002d5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1050,6 +1050,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11674. oneByteBuf in CryptoInputStream and CryptoOutputStream
should be non static. (Sean Busbey via yliu)
+ HADOOP-11670. Regression: s3a auth setup broken. (Adam Budde via stevel)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64443490/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index e7462dc..3486dfb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.fs.s3a;
-
public class Constants {
+ // s3 access key
+ public static final String ACCESS_KEY = "fs.s3a.access.key";
+
+ // s3 secret key
+ public static final String SECRET_KEY = "fs.s3a.secret.key";
// number of simultaneous connections to s3
public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64443490/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 1a30d6f..91a606c 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -32,8 +32,6 @@ import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.fs.s3.S3Credentials;
-
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
@@ -159,12 +157,22 @@ public class S3AFileSystem extends FileSystem {
this.getWorkingDirectory());
// Try to get our credentials or just connect anonymously
- S3Credentials s3Credentials = new S3Credentials();
- s3Credentials.initialize(name, conf);
+ String accessKey = conf.get(ACCESS_KEY, null);
+ String secretKey = conf.get(SECRET_KEY, null);
+
+ String userInfo = name.getUserInfo();
+ if (userInfo != null) {
+ int index = userInfo.indexOf(':');
+ if (index != -1) {
+ accessKey = userInfo.substring(0, index);
+ secretKey = userInfo.substring(index + 1);
+ } else {
+ accessKey = userInfo;
+ }
+ }
AWSCredentialsProviderChain credentials = new AWSCredentialsProviderChain(
- new BasicAWSCredentialsProvider(s3Credentials.getAccessKey(),
- s3Credentials.getSecretAccessKey()),
+ new BasicAWSCredentialsProvider(accessKey, secretKey),
new InstanceProfileCredentialsProvider(),
new AnonymousAWSCredentialsProvider()
);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64443490/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index bf62634..e0389c0 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -141,12 +141,12 @@ If you do any of these: change your credentials immediately!
### Authentication properties
<property>
- <name>fs.s3a.awsAccessKeyId</name>
+ <name>fs.s3a.access.key</name>
<description>AWS access key ID. Omit for Role-based authentication.</description>
</property>
<property>
- <name>fs.s3a.awsSecretAccessKey</name>
+ <name>fs.s3a.secret.key</name>
<description>AWS secret key. Omit for Role-based authentication.</description>
</property>
@@ -411,13 +411,13 @@ Example:
</property>
<property>
- <name>fs.s3a.awsAccessKeyId</name>
+ <name>fs.s3a.access.key</name>
<description>AWS access key ID. Omit for Role-based authentication.</description>
- <value>DONOTPCOMMITTHISKEYTOSCM</value>
+ <value>DONOTCOMMITTHISKEYTOSCM</value>
</property>
<property>
- <name>fs.s3a.awsSecretAccessKey</name>
+ <name>fs.s3a.secret.key</name>
<description>AWS secret key. Omit for Role-based authentication.</description>
<value>DONOTEVERSHARETHISSECRETKEY!</value>
</property>