You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by eh...@apache.org on 2018/08/13 10:58:33 UTC
hadoop git commit: HADOOP-15645.
ITestS3GuardToolLocal.testDiffCommand fails if bucket has per-bucket binding
to DDB. Contributed by Steve Loughran.
Repository: hadoop
Updated Branches:
refs/heads/trunk 475bff6e8 -> a13929ddc
HADOOP-15645. ITestS3GuardToolLocal.testDiffCommand fails if bucket has per-bucket binding to DDB. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a13929dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a13929dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a13929dd
Branch: refs/heads/trunk
Commit: a13929ddcb3b90044350ae1c23a1150e8b4b975b
Parents: 475bff6
Author: Ewan Higgs <ew...@wdc.com>
Authored: Mon Aug 13 12:57:45 2018 +0200
Committer: Ewan Higgs <ew...@wdc.com>
Committed: Mon Aug 13 12:57:45 2018 +0200
----------------------------------------------------------------------
.../fs/s3a/s3guard/AbstractS3GuardToolTestBase.java | 13 ++++++++++---
.../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java | 7 ++++---
.../hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java | 11 +++++++++++
3 files changed, 25 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a13929dd/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 9185fc5..96aac15 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -56,6 +56,7 @@ import static org.apache.hadoop.fs.s3a.Constants.METADATASTORE_AUTHORITATIVE;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_TABLE_NAME_KEY;
import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
+import static org.apache.hadoop.fs.s3a.S3AUtils.clearBucketOption;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -142,12 +143,14 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
public void setup() throws Exception {
super.setup();
S3ATestUtils.assumeS3GuardState(true, getConfiguration());
- ms = getFileSystem().getMetadataStore();
+ S3AFileSystem fs = getFileSystem();
+ ms = fs.getMetadataStore();
// Also create a "raw" fs without any MetadataStore configured
Configuration conf = new Configuration(getConfiguration());
- URI fsUri = getFileSystem().getUri();
+ clearBucketOption(conf, fs.getBucket(), S3_METADATA_STORE_IMPL);
conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
+ URI fsUri = fs.getUri();
S3AUtils.setBucketOption(conf,fsUri.getHost(),
METADATASTORE_AUTHORITATIVE,
S3GUARD_METASTORE_NULL);
@@ -394,13 +397,17 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
}
@Test
- public void testDiffCommand() throws Exception {
+ public void
+ testDiffCommand() throws Exception {
S3AFileSystem fs = getFileSystem();
ms = getMetadataStore();
Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
Path testPath = path("test-diff");
+ // clean up through the store and behind it.
+ fs.delete(testPath, true);
+ rawFs.delete(testPath, true);
mkdirs(testPath, true, true);
Path msOnlyPath = new Path(testPath, "ms_only");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a13929dd/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index 1a59bf1..c96cbd0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -50,9 +50,10 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
@Override
public void setup() throws Exception {
super.setup();
- Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
- getConfiguration().get(Constants.S3_METADATA_STORE_IMPL).equals(
- Constants.S3GUARD_METASTORE_DYNAMO));
+ MetadataStore ms = getMetadataStore();
+ Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard;"
+ + "Store is " + (ms == null ? "none" : ms.toString()),
+ ms instanceof DynamoDBMetadataStore);
}
// Check the existence of a given DynamoDB table.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a13929dd/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
index f5c4b03..1ee3cde 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -31,6 +31,8 @@ import java.util.concurrent.Callable;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.StringUtils;
+
+import org.junit.Assume;
import org.junit.Test;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -52,6 +54,15 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
"-force", "-verbose"};
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ MetadataStore ms = getMetadataStore();
+ Assume.assumeTrue("Test only applies when a local store is used for S3Guard;"
+ + "Store is " + (ms == null ? "none" : ms.toString()),
+ ms instanceof LocalMetadataStore);
+ }
+
@Test
public void testImportCommand() throws Exception {
S3AFileSystem fs = getFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org