You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2020/11/23 14:08:36 UTC
[hadoop] branch trunk updated: HADOOP-17343. Upgrade AWS SDK to
1.11.901 (#2468)
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new fb79be9 HADOOP-17343. Upgrade AWS SDK to 1.11.901 (#2468)
fb79be9 is described below
commit fb79be932c28a67ed01576252f44eae072fef7f0
Author: Steve Loughran <st...@cloudera.com>
AuthorDate: Mon Nov 23 14:08:12 2020 +0000
HADOOP-17343. Upgrade AWS SDK to 1.11.901 (#2468)
Contributed by Steve Loughran.
---
LICENSE-binary | 2 +-
hadoop-project/pom.xml | 2 +-
.../src/site/markdown/tools/hadoop-aws/testing.md | 96 ++++++++++++++++++++--
3 files changed, 91 insertions(+), 9 deletions(-)
diff --git a/LICENSE-binary b/LICENSE-binary
index dedd83a..4a4b953 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -215,7 +215,7 @@ com.aliyun:aliyun-java-sdk-ecs:4.2.0
com.aliyun:aliyun-java-sdk-ram:3.0.0
com.aliyun:aliyun-java-sdk-sts:3.0.0
com.aliyun.oss:aliyun-sdk-oss:3.4.1
-com.amazonaws:aws-java-sdk-bundle:1.11.563
+com.amazonaws:aws-java-sdk-bundle:1.11.901
com.cedarsoftware:java-util:1.9.0
com.cedarsoftware:json-io:2.5.1
com.fasterxml.jackson.core:jackson-annotations:2.9.9
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 3132281..2fe07c3 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -186,7 +186,7 @@
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<surefire.fork.timeout>900</surefire.fork.timeout>
- <aws-java-sdk.version>1.11.563</aws-java-sdk.version>
+ <aws-java-sdk.version>1.11.901</aws-java-sdk.version>
<hsqldb.version>2.3.4</hsqldb.version>
<frontend-maven-plugin.version>1.6</frontend-maven-plugin.version>
<jasmine-maven-plugin.version>2.1</jasmine-maven-plugin.version>
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index e973044..2046c4a 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -1499,36 +1499,69 @@ export HADOOP_OPTIONAL_TOOLS="hadoop-aws"
Run some basic s3guard commands as well as file operations.
```bash
-export BUCKET=s3a://example-bucket-name
+export BUCKETNAME=example-bucket-name
+export BUCKET=s3a://$BUCKETNAME
bin/hadoop s3guard bucket-info $BUCKET
+
+bin/hadoop s3guard uploads $BUCKET
+# repeat twice, once with "no" and once with "yes" as responses
+bin/hadoop s3guard uploads -abort $BUCKET
+
+# ---------------------------------------------------
+# assuming s3guard is enabled
+# if on pay-by-request, expect an error message and exit code of -1
bin/hadoop s3guard set-capacity $BUCKET
+
+# skip for PAY_PER_REQUEST
bin/hadoop s3guard set-capacity -read 15 -write 15 $BUCKET
-bin/hadoop s3guard uploads $BUCKET
+
+bin/hadoop s3guard bucket-info -guarded $BUCKET
+
bin/hadoop s3guard diff $BUCKET/
bin/hadoop s3guard prune -minutes 10 $BUCKET/
-bin/hadoop s3guard import $BUCKET/
+bin/hadoop s3guard import -verbose $BUCKET/
+bin/hadoop s3guard authoritative -verbose $BUCKET
+
+# ---------------------------------------------------
+# root filesystem operatios
+# ---------------------------------------------------
+
bin/hadoop fs -ls $BUCKET/
+# assuming file is not yet created, expect error and status code of 1
bin/hadoop fs -ls $BUCKET/file
+
+# exit code of 0 even when path doesn't exist
bin/hadoop fs -rm -R -f $BUCKET/dir-no-trailing
bin/hadoop fs -rm -R -f $BUCKET/dir-trailing/
+
+# error because it is a directory
bin/hadoop fs -rm $BUCKET/
+
bin/hadoop fs -touchz $BUCKET/file
-# expect I/O error as root dir is not empty
+# expect I/O error as it is the root directory
bin/hadoop fs -rm -r $BUCKET/
+
+# succeeds
bin/hadoop fs -rm -r $BUCKET/\*
-# now success
-bin/hadoop fs -rm -r $BUCKET/
+
+# ---------------------------------------------------
+# File operations
+# ---------------------------------------------------
bin/hadoop fs -mkdir $BUCKET/dir-no-trailing
-# fails with S3Guard
+# used to fail with S3Guard
bin/hadoop fs -mkdir $BUCKET/dir-trailing/
bin/hadoop fs -touchz $BUCKET/file
bin/hadoop fs -ls $BUCKET/
bin/hadoop fs -mv $BUCKET/file $BUCKET/file2
# expect "No such file or directory"
bin/hadoop fs -stat $BUCKET/file
+
+# expect success
bin/hadoop fs -stat $BUCKET/file2
+
+# expect "file exists"
bin/hadoop fs -mkdir $BUCKET/dir-no-trailing
bin/hadoop fs -mv $BUCKET/file2 $BUCKET/dir-no-trailing
bin/hadoop fs -stat $BUCKET/dir-no-trailing/file2
@@ -1545,8 +1578,57 @@ bin/hadoop fs -checksum $BUCKET/dir-no-trailing/file2
# expect "etag" + a long string
bin/hadoop fs -D fs.s3a.etag.checksum.enabled=true -checksum $BUCKET/dir-no-trailing/file2
bin/hadoop fs -expunge -immediate -fs $BUCKET
+
+# ---------------------------------------------------
+# Delegation Token support
+# ---------------------------------------------------
+
+# failure unless delegation tokens are enabled
bin/hdfs fetchdt --webservice $BUCKET secrets.bin
+# success
bin/hdfs fetchdt -D fs.s3a.delegation.token.binding=org.apache.hadoop.fs.s3a.auth.delegation.SessionTokenBinding --webservice $BUCKET secrets.bin
+bin/hdfs fetchdt -print secrets.bin
+
+# expect warning "No TokenRenewer defined for token kind S3ADelegationToken/Session"
+bin/hdfs fetchdt -renew secrets.bin
+
+# ---------------------------------------------------
+# Directory markers
+# ---------------------------------------------------
+
+# require success
+bin/hadoop s3guard bucket-info -markers aware $BUCKET
+# expect failure unless bucket policy is keep
+bin/hadoop s3guard bucket-info -markers keep $BUCKET/path
+
+# you may need to set this on a per-bucket basis if you have already been
+# playing with options
+bin/hadoop s3guard -D fs.s3a.directory.marker.retention=keep bucket-info -markers keep $BUCKET/path
+bin/hadoop s3guard -D fs.s3a.bucket.$BUCKETNAME.directory.marker.retention=keep bucket-info -markers keep $BUCKET/path
+
+# expect to see "Directory markers will be kept" messages and status code of "46"
+bin/hadoop fs -D fs.s3a.bucket.$BUCKETNAME.directory.marker.retention=keep -mkdir $BUCKET/p1
+bin/hadoop fs -D fs.s3a.bucket.$BUCKETNAME.directory.marker.retention=keep -mkdir $BUCKET/p1/p2
+bin/hadoop fs -D fs.s3a.bucket.$BUCKETNAME.directory.marker.retention=keep -touchz $BUCKET/p1/p2/file
+
+# expect failure as markers will be found for /p1/ and /p1/p2/
+bin/hadoop s3guard markers -audit -verbose $BUCKET
+
+# clean will remove markers
+bin/hadoop s3guard markers -clean -verbose $BUCKET
+
+# expect success and exit code of 0
+bin/hadoop s3guard markers -audit -verbose $BUCKET
+
+# ---------------------------------------------------
+# S3 Select on Landsat
+# ---------------------------------------------------
+
+export LANDSATGZ=s3a://landsat-pds/scene_list.gz
+
+bin/hadoop s3guard select -header use -compression gzip $LANDSATGZ \
+ "SELECT s.entityId,s.cloudCover FROM S3OBJECT s WHERE s.cloudCover < '0.0' LIMIT 100"
+
```
### Other tests
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org