You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/03/01 10:55:55 UTC
hadoop git commit: HADOOP-14125 s3guard tool tests aren't isolated;
can't run in parallel. Contributed by Sean Mackrory.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 f41d9b024 -> 0942c9f96
HADOOP-14125 s3guard tool tests aren't isolated; can't run in parallel.
Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0942c9f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0942c9f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0942c9f9
Branch: refs/heads/HADOOP-13345
Commit: 0942c9f96d401936774ed659a629945bedbc8276
Parents: f41d9b0
Author: Steve Loughran <st...@apache.org>
Authored: Wed Mar 1 10:55:19 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Mar 1 10:55:19 2017 +0000
----------------------------------------------------------------------
.../fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java | 2 +-
.../hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java | 14 +++++---------
.../hadoop/fs/s3a/s3guard/TestS3GuardTool.java | 16 +++++++++-------
3 files changed, 15 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942c9f9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index fb0efd1..c69bade 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -64,7 +64,7 @@ public class ITestS3GuardToolDynamoDB extends S3GuardToolTestBase {
public void testDynamoDBInitDestroyCycle() throws IOException,
InterruptedException {
String testTableName = "testDynamoDBInitDestroy" + new Random().nextInt();
- String testS3Url = getTestPath(testTableName);
+ String testS3Url = path(testTableName).toString();
S3AFileSystem fs = getFs();
DynamoDB db = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942c9f9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
index 666ec8e..0217a87 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardToolTestBase.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
@@ -39,7 +40,7 @@ import static org.junit.Assert.assertEquals;
/**
* Common functionality for S3GuardTool test cases.
*/
-public abstract class S3GuardToolTestBase {
+public abstract class S3GuardToolTestBase extends AbstractS3ATestBase {
protected static final String OWNER = "hdfs";
@@ -59,11 +60,6 @@ public abstract class S3GuardToolTestBase {
return fs;
}
- /** Get test path of s3. */
- protected String getTestPath(String path) {
- return fs.qualify(new Path(path)).toString();
- }
-
protected abstract MetadataStore newMetadataStore();
@Before
@@ -125,7 +121,7 @@ public abstract class S3GuardToolTestBase {
private void testPruneCommand(Configuration cmdConf, String[] args)
throws Exception {
- Path parent = new Path(getTestPath("/prune-cli"));
+ Path parent = path("prune-cli");
try {
fs.mkdirs(parent);
@@ -148,7 +144,7 @@ public abstract class S3GuardToolTestBase {
@Test
public void testPruneCommandCLI() throws Exception {
- String testPath = getTestPath("testPruneCommandCLI");
+ String testPath = path("testPruneCommandCLI").toString();
testPruneCommand(fs.getConf(), new String[]{"prune", "-S", "1", testPath});
}
@@ -156,7 +152,7 @@ public abstract class S3GuardToolTestBase {
public void testPruneCommandConf() throws Exception {
conf.setLong(Constants.S3GUARD_CLI_PRUNE_AGE,
TimeUnit.SECONDS.toMillis(1));
- String testPath = getTestPath("testPruneCommandConf");
+ String testPath = path("testPruneCommandConf").toString();
testPruneCommand(conf, new String[]{"prune", testPath});
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0942c9f9/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
index 6a39d7b..09c6f6b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestS3GuardTool.java
@@ -51,8 +51,9 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
public void testImportCommand() throws IOException {
S3AFileSystem fs = getFs();
MetadataStore ms = getMetadataStore();
- fs.mkdirs(new Path("/test"));
- Path dir = new Path("/test/a");
+ Path parent = path("test-import");
+ fs.mkdirs(parent);
+ Path dir = new Path(parent, "a");
fs.mkdirs(dir);
for (int i = 0; i < 10; i++) {
String child = String.format("file-%d", i);
@@ -65,10 +66,10 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
cmd.setMetadataStore(ms);
assertEquals("Import command did not exit successfully - see output",
- SUCCESS, cmd.run(new String[]{"import", getTestPath("/test/a")}));
+ SUCCESS, cmd.run(new String[]{"import", dir.toString()}));
DirListingMetadata children =
- ms.listChildren(new Path(getTestPath("/test/a")));
+ ms.listChildren(dir);
assertEquals("Unexpected number of paths imported", 10, children
.getListing().size());
// assertTrue(children.isAuthoritative());
@@ -81,8 +82,8 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
Set<Path> filesOnS3 = new HashSet<>(); // files on S3.
Set<Path> filesOnMS = new HashSet<>(); // files on metadata store.
- String testPath = getTestPath("/test-diff");
- mkdirs(new Path(testPath), true, true);
+ Path testPath = path("test-diff");
+ mkdirs(testPath, true, true);
Path msOnlyPath = new Path(testPath, "ms_only");
mkdirs(msOnlyPath, false, true);
@@ -107,7 +108,8 @@ public class TestS3GuardTool extends S3GuardToolTestBase {
Diff cmd = new Diff(fs.getConf());
cmd.setMetadataStore(ms);
assertEquals("Diff command did not exit successfully - see output", SUCCESS,
- cmd.run(new String[]{"diff", "-m", "local://metadata", testPath}, out));
+ cmd.run(new String[]{"diff", "-m", "local://metadata",
+ testPath.toString()}, out));
Set<Path> actualOnS3 = new HashSet<>();
Set<Path> actualOnMS = new HashSet<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org