You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/12/14 18:58:30 UTC
[28/50] [abbrv] hadoop git commit: HADOOP-15845. Require explicit URI
on CLI for s3guard init and destroy. Contributed by Gabor Bota.
HADOOP-15845. Require explicit URI on CLI for s3guard init and destroy. Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a25bbe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a25bbe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a25bbe9
Branch: refs/heads/HDFS-12943
Commit: 1a25bbe9ec7155712a82b157a8379826f8c79d4b
Parents: 21c9080
Author: Sean Mackrory <ma...@apache.org>
Authored: Mon Dec 10 17:10:03 2018 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Tue Dec 11 08:33:13 2018 -0700
----------------------------------------------------------------------
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 32 +++++++++++++++++---
.../s3guard/AbstractS3GuardToolTestBase.java | 13 ++++++++
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 3 +-
3 files changed, 43 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a25bbe9/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 8bef357..70ea1f4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -218,7 +218,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
format.addOptionWithValue(SECONDS_FLAG);
}
- protected void checkMetadataStoreUri(List<String> paths) throws IOException {
+ protected void checkIfS3BucketIsGuarded(List<String> paths)
+ throws IOException {
// be sure that path is provided in params, so there's no IOoBE
String s3Path = "";
if(!paths.isEmpty()) {
@@ -240,6 +241,23 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
/**
+ * Check if bucket or DDB table name is set.
+ */
+ protected void checkBucketNameOrDDBTableNameProvided(List<String> paths) {
+ String s3Path = null;
+ if(!paths.isEmpty()) {
+ s3Path = paths.get(0);
+ }
+
+ String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
+
+ if(metadataStoreUri == null && s3Path == null) {
+ throw invalidArgs("S3 bucket url or DDB table name have to be provided "
+ + "explicitly to use " + getName() + " command.");
+ }
+ }
+
+ /**
* Parse metadata store from command line option or HDFS configuration.
*
* @param forceCreate override the auto-creation setting to true.
@@ -433,6 +451,12 @@ public abstract class S3GuardTool extends Configured implements Tool {
@Override
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
+ try {
+ checkBucketNameOrDDBTableNameProvided(paths);
+ } catch (ExitUtil.ExitException e) {
+ errorln(USAGE);
+ throw e;
+ }
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (readCap != null && !readCap.isEmpty()) {
@@ -521,7 +545,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
Map<String, String> options = new HashMap<>();
- checkMetadataStoreUri(paths);
+ checkIfS3BucketIsGuarded(paths);
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (StringUtils.isNotEmpty(readCap)) {
@@ -592,14 +616,14 @@ public abstract class S3GuardTool extends Configured implements Tool {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
try {
+ checkBucketNameOrDDBTableNameProvided(paths);
+ checkIfS3BucketIsGuarded(paths);
parseDynamoDBRegion(paths);
} catch (ExitUtil.ExitException e) {
errorln(USAGE);
throw e;
}
- checkMetadataStoreUri(paths);
-
try {
initMetadataStore(false);
} catch (FileNotFoundException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a25bbe9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 9378358..48c60ce 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -381,6 +381,19 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
}
}
+ @Test
+ public void testDestroyFailsIfNoBucketNameOrDDBTableSet()
+ throws Exception {
+ intercept(ExitUtil.ExitException.class,
+ () -> run(S3GuardTool.Destroy.NAME));
+ }
+
+ @Test
+ public void testInitFailsIfNoBucketNameOrDDBTableSet() throws Exception {
+ intercept(ExitUtil.ExitException.class,
+ () -> run(S3GuardTool.Init.NAME));
+ }
+
/**
* Get the test CSV file; assume() that it is not modified (i.e. we haven't
* switched to a new storage infrastructure where the bucket is no longer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a25bbe9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index 65e2619..97173fe 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -126,7 +126,8 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
String[] argsR = new String[]{
cmdR.getName(),
- "-tag", tagMapToStringParams(tagMap)
+ "-tag", tagMapToStringParams(tagMap),
+ getFileSystem().getBucket()
};
// run
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org