You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by fa...@apache.org on 2017/02/07 02:02:32 UTC
hadoop git commit: HADOOP-14013 S3Guard: fix multi-bucket integration
tests. Contributed by Aaron Fabbri
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 758b30ee8 -> a7e6dbef2
HADOOP-14013 S3Guard: fix multi-bucket integration tests. Contributed by Aaron Fabbri
with input from Mingliang Liu and Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7e6dbef
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7e6dbef
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7e6dbef
Branch: refs/heads/HADOOP-13345
Commit: a7e6dbef27331bdcd52223f5b8f48e5645179b8b
Parents: 758b30e
Author: Aaron Fabbri <fa...@apache.org>
Authored: Mon Feb 6 17:47:40 2017 -0800
Committer: Aaron Fabbri <fa...@apache.org>
Committed: Mon Feb 6 17:47:40 2017 -0800
----------------------------------------------------------------------
.../site/markdown/tools/hadoop-aws/s3guard.md | 12 ++++++
.../hadoop-aws/src/test/resources/core-site.xml | 40 ++++++++++++++++----
2 files changed, 45 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7e6dbef/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
index 05701fb..dac3d19 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
@@ -219,6 +219,18 @@ Attempting to perform more IO than the capacity requested simply throttles the
IO; small capacity numbers are recommended when initially experimenting
with S3Guard.
+## Credentials
+
+The DynamoDB metadata store takes advantage of the fact that the DynamoDB
+service uses uses the same authentication mechanisms as S3. With S3Guard,
+DynamoDB doesn't have any dedicated authentication configuration; it gets its
+credentials from the S3A client that is using it.
+
+The existing S3 authentication mechanisms can be used, except for one
+exception. Credentials placed in URIs are not supported for S3Guard. The
+reason is that providing login details in filesystem URIs is considered
+unsafe and thus deprecated.
+
## S3Guard Command Line Interface (CLI)
Note that in some cases an endpoint or a s3a:// URI can be provided.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7e6dbef/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index d76c4c8..fa7e270 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -30,19 +30,45 @@
<final>true</final>
</property>
- <!-- Uncomment these two properties to run integration tests with
- local metadata store (S3Guard).
+ <property>
+ <name>fs.s3a.bucket.landsat-pds.endpoint</name>
+ <value>${central.endpoint}</value>
+ <description>The endpoint for s3a://landsat-pds URLs</description>
+ </property>
+ <!-- Make sure S3Guard is disabled for read-only bucket tests. -->
<property>
- <name>fs.s3a.metadatastore.authoritative</name>
- <value>true</value>
+ <name>fs.s3a.bucket.landsat-pds.metadatastore.impl</name>
+ <value>${s3guard.null}</value>
+ <description>The read-only landsat-pds repository isn't
+ managed by s3guard</description>
</property>
+
+ <!-- Convenience definitions. -->
<property>
- <name>fs.s3a.metadatastore.impl</name>
- <value>org.apache.hadoop.fs.s3a.s3guard.LocalMetadataStore</value>
+ <name>s3guard.null</name>
+ <value>org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore</value>
</property>
- -->
+ <property>
+ <name>s3guard.dynamo</name>
+ <value>org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore</value>
+ </property>
+
+ <property>
+ <name>frankfurt.endpoint</name>
+ <value>s3.eu-central-1.amazonaws.com</value>
+ </property>
+
+ <property>
+ <name>ireland.endpoint</name>
+ <value>s3-eu-west-1.amazonaws.com</value>
+ </property>
+
+ <property>
+ <name>central.endpoint</name>
+ <value>s3.amazonaws.com</value>
+ </property>
<!-- Scale integration tests may time out on slower connections
you can reduce the operation count like so to mitigate this.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org