You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/05/30 21:02:14 UTC
[47/50] [abbrv] hadoop git commit: HADOOP-15480
AbstractS3GuardToolTestBase.testDiffCommand fails when using dynamo (Gabor
Bota)
HADOOP-15480 AbstractS3GuardToolTestBase.testDiffCommand fails when using dynamo (Gabor Bota)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d8597b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d8597b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d8597b1
Branch: refs/heads/HDDS-48
Commit: 6d8597b1939a301bed758bf617b8c8b611e1857f
Parents: c6bdbf1
Author: Aaron Fabbri <fa...@apache.org>
Authored: Tue May 29 19:20:22 2018 -0700
Committer: Hanisha Koneru <ha...@apache.org>
Committed: Wed May 30 14:00:26 2018 -0700
----------------------------------------------------------------------
.../s3guard/AbstractS3GuardToolTestBase.java | 37 +++++++++++++-------
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 5 ---
.../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 5 ---
3 files changed, 25 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d8597b1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 2b43810..7d75f52 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintStream;
+import java.net.URI;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
@@ -32,6 +33,8 @@ import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.util.StopWatch;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.FileSystem;
import org.junit.Assume;
import org.junit.Test;
@@ -48,6 +51,8 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_METASTORE_NULL;
+import static org.apache.hadoop.fs.s3a.Constants.S3_METADATA_STORE_IMPL;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -65,6 +70,7 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
private static final int PRUNE_MAX_AGE_SECS = 2;
private MetadataStore ms;
+ private S3AFileSystem rawFs;
protected static void expectResult(int expected,
String message,
@@ -129,28 +135,34 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
return ms;
}
- protected abstract MetadataStore newMetadataStore();
-
@Override
public void setup() throws Exception {
super.setup();
S3ATestUtils.assumeS3GuardState(true, getConfiguration());
- ms = newMetadataStore();
- ms.initialize(getFileSystem());
+ ms = getFileSystem().getMetadataStore();
+
+ // Also create a "raw" fs without any MetadataStore configured
+ Configuration conf = new Configuration(getConfiguration());
+ conf.set(S3_METADATA_STORE_IMPL, S3GUARD_METASTORE_NULL);
+ URI fsUri = getFileSystem().getUri();
+ rawFs = (S3AFileSystem) FileSystem.newInstance(fsUri, conf);
}
@Override
public void teardown() throws Exception {
super.teardown();
IOUtils.cleanupWithLogger(LOG, ms);
+ IOUtils.closeStream(rawFs);
}
protected void mkdirs(Path path, boolean onS3, boolean onMetadataStore)
throws IOException {
+ Preconditions.checkArgument(onS3 || onMetadataStore);
+ // getFileSystem() returns an fs with MetadataStore configured
+ S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs;
if (onS3) {
- getFileSystem().mkdirs(path);
- }
- if (onMetadataStore) {
+ fs.mkdirs(path);
+ } else if (onMetadataStore) {
S3AFileStatus status = new S3AFileStatus(true, path, OWNER);
ms.put(new PathMetadata(status));
}
@@ -178,13 +190,14 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
*/
protected void createFile(Path path, boolean onS3, boolean onMetadataStore)
throws IOException {
+ Preconditions.checkArgument(onS3 || onMetadataStore);
+ // getFileSystem() returns an fs with MetadataStore configured
+ S3AFileSystem fs = onMetadataStore ? getFileSystem() : rawFs;
if (onS3) {
- ContractTestUtils.touch(getFileSystem(), path);
- }
-
- if (onMetadataStore) {
+ ContractTestUtils.touch(fs, path);
+ } else if (onMetadataStore) {
S3AFileStatus status = new S3AFileStatus(100L, System.currentTimeMillis(),
- getFileSystem().qualify(path), 512L, "hdfs");
+ fs.qualify(path), 512L, "hdfs");
putFile(ms, status);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d8597b1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index 821bba5..1a59bf1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -48,11 +48,6 @@ import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.*;
public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
@Override
- protected MetadataStore newMetadataStore() {
- return new DynamoDBMetadataStore();
- }
-
- @Override
public void setup() throws Exception {
super.setup();
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d8597b1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
index a40c7a5..f5c4b03 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -52,11 +52,6 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
private static final String[] ABORT_FORCE_OPTIONS = new String[] {"-abort",
"-force", "-verbose"};
- @Override
- protected MetadataStore newMetadataStore() {
- return new LocalMetadataStore();
- }
-
@Test
public void testImportCommand() throws Exception {
S3AFileSystem fs = getFileSystem();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org