You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2016/06/09 20:01:23 UTC
[3/3] hadoop git commit: HADOOP-12537 S3A to support Amazon STS
temporary credentials. Contributed by Sean Mackrory.
HADOOP-12537 S3A to support Amazon STS temporary credentials. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/31ffaf76
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/31ffaf76
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/31ffaf76
Branch: refs/heads/trunk
Commit: 31ffaf76f2b6e1fd2a141daa4daaebdfecefe727
Parents: 9378d94
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jun 9 20:58:30 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jun 9 21:00:47 2016 +0100
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 5 +
hadoop-project/pom.xml | 8 +-
hadoop-tools/hadoop-aws/pom.xml | 5 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 3 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 3 +
.../s3a/CredentialInitializationException.java | 46 ++++++
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 70 +++++++++
.../src/site/markdown/tools/hadoop-aws/index.md | 71 ++++++++-
.../fs/s3a/TestS3ATemporaryCredentials.java | 150 +++++++++++++++++++
9 files changed, 357 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 8bb27ea..39b7132 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -806,6 +806,11 @@
</property>
<property>
+ <name>fs.s3a.session.token</name>
+ <description>The session token used with temporary credentials. Used only with provider org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider.</description>
+</property>
+
+<property>
<name>fs.s3a.connection.maximum</name>
<value>15</value>
<description>Controls the maximum number of simultaneous connections to S3.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 2b6b162..4c618a1 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -116,6 +116,7 @@
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
+ <aws-java-sdk.version>1.10.6</aws-java-sdk.version>
</properties>
<dependencyManagement>
@@ -690,7 +691,12 @@
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
- <version>1.10.6</version>
+ <version>${aws-java-sdk.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-sts</artifactId>
+ <version>${aws-java-sdk.version}</version>
</dependency>
<dependency>
<groupId>org.apache.mina</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index c95f1e6..7c25e60 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -231,6 +231,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-sts</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
index 3a5ee8c..61be43f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BasicAWSCredentialsProvider.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.AWSCredentials;
@@ -49,7 +48,7 @@ public class BasicAWSCredentialsProvider implements AWSCredentialsProvider {
if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)) {
return new BasicAWSCredentials(accessKey, secretKey);
}
- throw new AmazonClientException(
+ throw new CredentialInitializationException(
"Access key or secret key is null");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index eb859ac..4abb550 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -41,6 +41,9 @@ public final class Constants {
public static final String AWS_CREDENTIALS_PROVIDER =
"fs.s3a.aws.credentials.provider";
+ // session token for when using TemporaryAWSCredentialsProvider
+ public static final String SESSION_TOKEN = "fs.s3a.session.token";
+
// number of simultaneous connections to s3
public static final String MAXIMUM_CONNECTIONS = "fs.s3a.connection.maximum";
public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
new file mode 100644
index 0000000..46655bc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/CredentialInitializationException.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonClientException;
+
+/**
+ * Exception which Hadoop's AWSCredentialsProvider implementations should
+ * throw when there is a problem with the credential setup. This
+ * is a subclass of {@link AmazonClientException} which sets
+ * {@link #isRetryable()} to false, so as to fail fast.
+ */
+public class CredentialInitializationException extends AmazonClientException {
+ public CredentialInitializationException(String message, Throwable t) {
+ super(message, t);
+ }
+
+ public CredentialInitializationException(String message) {
+ super(message);
+ }
+
+ /**
+ * This exception is not going to go away if you try calling it again.
+ * @return false, always.
+ */
+ @Override
+ public boolean isRetryable() {
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
new file mode 100644
index 0000000..190f7bc
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/TemporaryAWSCredentialsProvider.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicSessionCredentials;
+import com.amazonaws.auth.AWSCredentials;
+import org.apache.commons.lang.StringUtils;
+
+import java.net.URI;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Support session credentials for authenticating with AWS.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Stable
+public class TemporaryAWSCredentialsProvider implements AWSCredentialsProvider {
+
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
+ private final String accessKey;
+ private final String secretKey;
+ private final String sessionToken;
+
+ public TemporaryAWSCredentialsProvider(URI uri, Configuration conf) {
+ this.accessKey = conf.get(ACCESS_KEY, null);
+ this.secretKey = conf.get(SECRET_KEY, null);
+ this.sessionToken = conf.get(SESSION_TOKEN, null);
+ }
+
+ public AWSCredentials getCredentials() {
+ if (!StringUtils.isEmpty(accessKey) && !StringUtils.isEmpty(secretKey)
+ && !StringUtils.isEmpty(sessionToken)) {
+ return new BasicSessionCredentials(accessKey, secretKey, sessionToken);
+ }
+ throw new CredentialInitializationException(
+ "Access key, secret key or session token is unset");
+ }
+
+ @Override
+ public void refresh() {}
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 4086bc0..606275c 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -201,6 +201,46 @@ If you do any of these: change your credentials immediately!
</description>
</property>
+ <property>
+ <name>fs.s3a.session.token</name>
+ <description>Session token, when using org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider as the providers.</description>
+ </property>
+
+#### Authentication methods
+
+The standard way to authenticate is with an access key and secret key using the
+properties above. You can also avoid configuring credentials if the EC2
+instances in your cluster are configured with IAM instance profiles that grant
+the appropriate S3 access.
+
+A temporary set of credentials can also be obtained from Amazon STS; these
+consist of an access key, a secret key, and a session token. To use these
+temporary credentials you must include the `aws-java-sdk-sts` JAR in your
+classpath (consult the POM for the current version) and set the
+`TemporaryAWSCredentialsProvider` class as the provider. The session key
+must be set in the property `fs.s3a.session.token` \u2014and the access and secret
+key properties to those of this temporary session.
+
+ <property>
+ <name>fs.s3a.aws.credentials.provider</name>
+ <value>org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>SESSION-ACCESS-KEY</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>SESSION-SECRET-KEY</value>
+ </property>
+
+ <property>
+ <name>fs.s3a.session.token</name>
+ <value>SECRET-SESSION-TOKEN</value>
+ </property>
+
#### Protecting the AWS Credentials in S3A
To protect the access/secret keys from prying eyes, it is recommended that you
@@ -605,6 +645,13 @@ Example:
<description>AWS secret key. Omit for IAM role-based authentication.</description>
<value>DONOTEVERSHARETHISSECRETKEY!</value>
</property>
+
+ <property>
+ <name>test.sts.endpoint</name>
+ <description>Specific endpoint to use for STS requests.</description>
+ <value>sts.amazonaws.com</value>
+ </property>
+
</configuration>
### File `contract-test-options.xml`
@@ -714,8 +761,30 @@ that the file `contract-test-options.xml` does not contain any
secret credentials itself. As the auth keys XML file is kept out of the
source code tree, it is not going to get accidentally committed.
-### Running Performance Tests against non-AWS storage infrastructures
+### Running Tests against non-AWS storage infrastructures
+
+### S3A session tests
+
+The test `TestS3ATemporaryCredentials` requests a set of temporary
+credentials from the STS service, then uses them to authenticate with S3.
+
+If an S3 implementation does not support STS, then the functional test
+cases must be disabled:
+
+ <property>
+ <name>test.fs.s3a.sts.enabled</name>
+ <value>false</value>
+ </property>
+
+These tests reqest a temporary set of credentials from the STS service endpoint.
+An alternate endpoint may be defined in `test.fs.s3a.sts.endpoint`.
+
+ <property>
+ <name>test.fs.s3a.sts.endpoint</name>
+ <value>https://sts.example.org/</value>
+ </property>
+The default is ""; meaning "use the amazon default value".
#### CSV Data source
http://git-wip-us.apache.org/repos/asf/hadoop/blob/31ffaf76/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
new file mode 100644
index 0000000..be5c599
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3ATemporaryCredentials.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProviderChain;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
+import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
+import com.amazonaws.services.securitytoken.model.GetSessionTokenResult;
+import com.amazonaws.services.securitytoken.model.Credentials;
+
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.apache.hadoop.fs.contract.s3a.S3AContract;
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * Tests use of temporary credentials (for example, AWS STS & S3).
+ * This test extends a class that "does things to the root directory", and
+ * should only be used against transient filesystems where you don't care about
+ * the data.
+ */
+public class TestS3ATemporaryCredentials extends AbstractFSContractTestBase {
+ public static final String TEST_STS_ENABLED = "test.fs.s3a.sts.enabled";
+ public static final String TEST_STS_ENDPOINT = "test.fs.s3a.sts.endpoint";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestS3ATemporaryCredentials.class);
+
+ private S3AFileSystem fs;
+
+
+ private static final String PROVIDER_CLASS =
+ "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider";
+
+ private static final long TEST_FILE_SIZE = 1024;
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new S3AContract(conf);
+ }
+
+ /**
+ * Test use of STS for requesting temporary credentials.
+ *
+ * The property test.sts.endpoint can be set to point this at different
+ * STS endpoints. This test will use the AWS credentials (if provided) for
+ * S3A tests to request temporary credentials, then attempt to use those
+ * credentials instead.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testSTS() throws IOException {
+ Configuration conf = getContract().getConf();
+ if (!conf.getBoolean(TEST_STS_ENABLED, true)) {
+ skip("STS functional tests disabled");
+ }
+
+ String parentAccessKey = conf.getTrimmed(ACCESS_KEY, null);
+ String parentSecretKey = conf.getTrimmed(SECRET_KEY, null);
+ String stsEndpoint = conf.getTrimmed(TEST_STS_ENDPOINT, "");
+ AWSCredentialsProviderChain parentCredentials;
+ parentCredentials = new AWSCredentialsProviderChain(
+ new BasicAWSCredentialsProvider(parentAccessKey, parentSecretKey),
+ new InstanceProfileCredentialsProvider()
+ );
+
+ AWSSecurityTokenServiceClient stsClient;
+ stsClient = new AWSSecurityTokenServiceClient(parentCredentials);
+ if (!stsEndpoint.isEmpty()) {
+ LOG.debug("STS Endpoint ={}", stsEndpoint);
+ stsClient.setEndpoint(stsEndpoint);
+ }
+ GetSessionTokenRequest sessionTokenRequest = new GetSessionTokenRequest();
+ sessionTokenRequest.setDurationSeconds(900);
+ GetSessionTokenResult sessionTokenResult;
+ sessionTokenResult = stsClient.getSessionToken(sessionTokenRequest);
+ Credentials sessionCreds = sessionTokenResult.getCredentials();
+
+ String childAccessKey = sessionCreds.getAccessKeyId();
+ conf.set(ACCESS_KEY, childAccessKey);
+ String childSecretKey = sessionCreds.getSecretAccessKey();
+ conf.set(SECRET_KEY, childSecretKey);
+ String sessionToken = sessionCreds.getSessionToken();
+ conf.set(SESSION_TOKEN, sessionToken);
+
+ conf.set(AWS_CREDENTIALS_PROVIDER, PROVIDER_CLASS);
+
+ try(S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ createAndVerifyFile(fs, path("testSTS"), TEST_FILE_SIZE);
+ }
+
+ // now create an invalid set of credentials by changing the session
+ // token
+ conf.set(SESSION_TOKEN, "invalid-" + sessionToken);
+ try (S3AFileSystem fs = S3ATestUtils.createTestFileSystem(conf)) {
+ createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
+ fail("Expected an access exception, but file access to "
+ + fs.getUri() + " was allowed: " + fs);
+ } catch (AWSS3IOException ex) {
+ LOG.info("Expected Exception: {}", ex.toString());
+ LOG.debug("Expected Exception: {}", ex, ex);
+ }
+ }
+
+ @Test
+ public void testTemporaryCredentialValidation() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY, "accesskey");
+ conf.set(SECRET_KEY, "secretkey");
+ conf.set(SESSION_TOKEN, "");
+ TemporaryAWSCredentialsProvider provider
+ = new TemporaryAWSCredentialsProvider(getFileSystem().getUri(), conf);
+ try {
+ AWSCredentials credentials = provider.getCredentials();
+ fail("Expected a CredentialInitializationException,"
+ + " got " + credentials);
+ } catch (CredentialInitializationException expected) {
+ // expected
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org