You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2018/09/12 15:13:19 UTC
hadoop git commit: HADOOP-15635. s3guard set-capacity command to fail
fast if bucket is unguarded. Contributed by Gabor Bota.
Repository: hadoop
Updated Branches:
refs/heads/trunk 47299ce23 -> 47b72c87e
HADOOP-15635. s3guard set-capacity command to fail fast if bucket is unguarded.
Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47b72c87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47b72c87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47b72c87
Branch: refs/heads/trunk
Commit: 47b72c87ebdd620af06e185c93ed44a9bd33def3
Parents: 47299ce
Author: Sean Mackrory <ma...@apache.org>
Authored: Wed Sep 12 08:07:10 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Wed Sep 12 09:12:38 2018 -0600
----------------------------------------------------------------------
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 14 ++++++++++++++
.../s3a/s3guard/AbstractS3GuardToolTestBase.java | 19 ++++++++++++++++++-
2 files changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47b72c87/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 19dc32a..22274ee 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -479,6 +479,20 @@ public abstract class S3GuardTool extends Configured implements Tool {
public int run(String[] args, PrintStream out) throws Exception {
List<String> paths = parseArgs(args);
Map<String, String> options = new HashMap<>();
+ String s3Path = paths.get(0);
+
+ // Check if DynamoDB url is set from arguments.
+ String metadataStoreUri = getCommandFormat().getOptValue(META_FLAG);
+ if(metadataStoreUri == null || metadataStoreUri.isEmpty()) {
+ // If not set, check if filesystem is guarded by creating an
+ // S3AFileSystem and check if hasMetadataStore is true
+ try (S3AFileSystem s3AFileSystem = (S3AFileSystem)
+ S3AFileSystem.newInstance(toUri(s3Path), getConf())){
+ Preconditions.checkState(s3AFileSystem.hasMetadataStore(),
+ "The S3 bucket is unguarded. " + getName()
+ + " can not be used on an unguarded bucket.");
+ }
+ }
String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (StringUtils.isNotEmpty(readCap)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47b72c87/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 96aac15..242f616 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -29,6 +29,7 @@ import java.net.URI;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
@@ -53,6 +54,7 @@ import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_CREATE_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
@@ -296,7 +298,7 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
}
@Test
- public void testSetCapacityFailFast() throws Exception{
+ public void testSetCapacityFailFastOnReadWriteOfZero() throws Exception{
Configuration conf = getConfiguration();
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, getFileSystem().getBucket());
@@ -313,6 +315,21 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
}
@Test
+ public void testSetCapacityFailFastIfNotGuarded() throws Exception{
+ Configuration conf = getConfiguration();
+ conf.set(S3GUARD_DDB_TABLE_NAME_KEY, UUID.randomUUID().toString());
+ conf.set(S3GUARD_DDB_TABLE_CREATE_KEY, Boolean.FALSE.toString());
+ conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
+
+ S3GuardTool.SetCapacity cmdR = new S3GuardTool.SetCapacity(conf);
+ String[] argsR = new String[]{cmdR.getName(),
+ "s3a://" + getFileSystem().getBucket()};
+
+ intercept(IllegalStateException.class, "unguarded",
+ () -> run(argsR));
+ }
+
+ @Test
public void testDestroyNoBucket() throws Throwable {
intercept(FileNotFoundException.class,
new Callable<Integer>() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org