You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/17 10:36:06 UTC
hadoop git commit: HADOOP-14777. S3Guard premerge changes: java 7
build & test tuning Contributed by Steve Loughran.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 67eb6202e -> f07c7aa3a
HADOOP-14777. S3Guard premerge changes: java 7 build & test tuning
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f07c7aa3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f07c7aa3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f07c7aa3
Branch: refs/heads/HADOOP-13345
Commit: f07c7aa3af3508ba9d7bea252f827301e40e177e
Parents: 67eb620
Author: Steve Loughran <st...@apache.org>
Authored: Thu Aug 17 11:35:24 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Aug 17 11:35:24 2017 +0100
----------------------------------------------------------------------
.../hadoop/fs/FileSystemContractBaseTest.java | 2 +-
.../fs/s3a/InconsistentAmazonS3Client.java | 10 +-
.../hadoop/fs/s3a/S3AInstrumentation.java | 2 +-
.../fs/s3a/s3guard/DescendantsIterator.java | 2 +-
.../fs/s3a/s3guard/DirListingMetadata.java | 8 +-
.../fs/s3a/s3guard/DynamoDBClientFactory.java | 4 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 14 +-
.../fs/s3a/s3guard/LocalMetadataStore.java | 2 +-
.../s3guard/MetadataStoreListFilesIterator.java | 2 +-
.../fs/s3a/s3guard/NullMetadataStore.java | 1 -
.../hadoop/fs/s3a/s3guard/PathMetadata.java | 2 +-
.../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 8 +-
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 153 ++++++++++---------
.../hadoop/fs/s3a/ITestS3ADelayedFNF.java | 11 +-
.../fs/s3a/ITestS3AFileSystemContract.java | 3 +-
.../hadoop/fs/s3a/ITestS3AMiscOperations.java | 11 +-
.../s3guard/AbstractS3GuardToolTestBase.java | 2 +-
.../s3a/s3guard/DynamoDBLocalClientFactory.java | 7 +-
.../s3a/s3guard/ITestS3GuardConcurrentOps.java | 5 +-
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 6 +-
.../fs/s3a/s3guard/ITestS3GuardToolLocal.java | 7 +-
.../fs/s3a/s3guard/TestDirListingMetadata.java | 10 +-
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 37 +++--
.../TestPathMetadataDynamoDBTranslation.java | 8 +-
24 files changed, 182 insertions(+), 135 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 5b2ba65..9d8cd64 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -751,7 +751,7 @@ public abstract class FileSystemContractBaseTest {
* renames empty dirs doesn't cause any regressions.
*/
public void testRenameEmptyToDirWithSamePrefixAllowed() throws Throwable {
- if (!renameSupported()) return;
+ assumeTrue(renameSupported());
Path parentdir = path("testRenameEmptyToDirWithSamePrefixAllowed");
fs.mkdirs(parentdir);
Path dest = path("testRenameEmptyToDirWithSamePrefixAllowedDest");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
index ba40d34..5e9cb3f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
@@ -168,9 +168,8 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
@Override
public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
- deleteObjectsRequest) throws AmazonClientException,
- AmazonServiceException
- {
+ deleteObjectsRequest)
+ throws AmazonClientException, AmazonServiceException {
for (DeleteObjectsRequest.KeyVersion keyVersion :
deleteObjectsRequest.getKeys()) {
registerDeleteObject(keyVersion.getKey(), deleteObjectsRequest
@@ -404,8 +403,9 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
private final List<S3ObjectSummary> customListing;
private final List<String> customPrefixes;
- public CustomObjectListing(ObjectListing rawListing,
- List<S3ObjectSummary> customListing, List<String> customPrefixes) {
+ CustomObjectListing(ObjectListing rawListing,
+ List<S3ObjectSummary> customListing,
+ List<String> customPrefixes) {
super();
this.customListing = customListing;
this.customPrefixes = customPrefixes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index c4b4866..da1fc5a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -345,7 +345,7 @@ public class S3AInstrumentation {
public MutableQuantiles lookupQuantiles(String name) {
MutableMetric metric = lookupMetric(name);
if (metric == null) {
- LOG.debug("No quantiles {}", name);
+ LOG.debug("No quantiles {}", name);
}
return (MutableQuantiles) metric;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java
index 262a6fa..dcee358 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DescendantsIterator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
index bcbdced..e5b4fb5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DirListingMetadata.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -222,11 +222,7 @@ public class DirListingMetadata {
Path childPath = childStatusToPathKey(childFileStatus);
PathMetadata newValue = new PathMetadata(childFileStatus);
PathMetadata oldValue = listMap.put(childPath, newValue);
- if (oldValue == null) {
- return true;
- } else {
- return !oldValue.equals(newValue);
- }
+ return oldValue == null || !oldValue.equals(newValue);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
index 5005bc0..66ada49 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBClientFactory.java
@@ -115,12 +115,12 @@ public interface DynamoDBClientFactory extends Configurable {
}
private static String validRegionsString() {
- final String DELIMITER = ", ";
+ final String delimiter = ", ";
Regions[] regions = Regions.values();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < regions.length; i++) {
if (i > 0) {
- sb.append(DELIMITER);
+ sb.append(delimiter);
}
sb.append(regions[i].getName());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 322361c..1bed03d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -481,8 +481,8 @@ public class DynamoDBMetadataStore implements MetadataStore {
parent = parent.getParent();
}
}
- return ancestry.values();
- }
+ return ancestry.values();
+ }
@Override
public void move(Collection<Path> pathsToDelete,
@@ -893,10 +893,13 @@ public class DynamoDBMetadataStore implements MetadataStore {
/**
* Wait for table being active.
+ * @param t table to block on.
+ * @throws IOException IO problems
+ * @throws InterruptedIOException if the wait was interrupted
*/
- private void waitForTableActive(Table table) throws IOException {
+ private void waitForTableActive(Table t) throws IOException {
try {
- table.waitForActive();
+ t.waitForActive();
} catch (InterruptedException e) {
LOG.warn("Interrupted while waiting for table {} in region {} active",
tableName, region, e);
@@ -910,7 +913,8 @@ public class DynamoDBMetadataStore implements MetadataStore {
* Create a table, wait for it to become active, then add the version
* marker.
* @param capacity capacity to provision
- * @throws IOException on an failure.
+ * @throws IOException on any failure.
+ * @throws InterruptedIOException if the wait was interrupted
*/
private void createTable(ProvisionedThroughput capacity) throws IOException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
index a6492f9..1ef8b0d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/LocalMetadataStore.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java
index 679f767..378d109 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreListFilesIterator.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
index 41dd61b..08ae89e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/NullMetadataStore.java
@@ -34,7 +34,6 @@ public class NullMetadataStore implements MetadataStore {
@Override
public void initialize(FileSystem fs) throws IOException {
- return;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
index d8cb447..2a0219e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/PathMetadata.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
index edd3830..7e4aec1 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
@@ -310,7 +310,6 @@ public final class S3Guard {
ms.put(pathMetas);
} catch (IOException ioe) {
LOG.error("MetadataStore#put() failure:", ioe);
- return;
}
}
@@ -429,8 +428,9 @@ public final class S3Guard {
}
private static void addMoveStatus(Collection<Path> srcPaths,
- Collection<PathMetadata> dstMetas, Path srcPath, FileStatus dstStatus)
- {
+ Collection<PathMetadata> dstMetas,
+ Path srcPath,
+ FileStatus dstStatus) {
srcPaths.add(srcPath);
dstMetas.add(new PathMetadata(dstStatus));
}
@@ -446,7 +446,7 @@ public final class S3Guard {
// multiple S3AFileSystem instances
Preconditions.checkNotNull(uri.getHost(), "Null host in " + uri);
- // I believe this should never fail, since there is a host?
+ // This should never fail, but is retained for completeness.
Preconditions.checkNotNull(uri.getScheme(), "Null scheme in " + uri);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index ea34734..be271ae 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -26,6 +26,7 @@ import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -85,9 +86,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
static final int INVALID_ARGUMENT = 1;
static final int ERROR = 99;
- protected S3AFileSystem s3a;
- protected MetadataStore ms;
- protected CommandFormat commandFormat;
+ private S3AFileSystem filesystem;
+ private MetadataStore store;
+ private final CommandFormat commandFormat;
private static final String META_FLAG = "meta";
private static final String DAYS_FLAG = "days";
@@ -118,11 +119,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
*/
abstract String getName();
- @VisibleForTesting
- public MetadataStore getMetadataStore() {
- return ms;
- }
-
/**
* Parse DynamoDB region from either -m option or a S3 path.
*
@@ -135,7 +131,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
*/
boolean parseDynamoDBRegion(List<String> paths) throws IOException {
Configuration conf = getConf();
- String fromCli = commandFormat.getOptValue(REGION_FLAG);
+ String fromCli = getCommandFormat().getOptValue(REGION_FLAG);
String fromConf = conf.get(S3GUARD_DDB_REGION_KEY);
boolean hasS3Path = !paths.isEmpty();
@@ -181,26 +177,26 @@ public abstract class S3GuardTool extends Configured implements Tool {
* @return a initialized metadata store.
*/
MetadataStore initMetadataStore(boolean forceCreate) throws IOException {
- if (ms != null) {
- return ms;
+ if (getStore() != null) {
+ return getStore();
}
Configuration conf;
- if (s3a == null) {
+ if (filesystem == null) {
conf = getConf();
} else {
- conf = s3a.getConf();
+ conf = filesystem.getConf();
}
- String metaURI = commandFormat.getOptValue(META_FLAG);
+ String metaURI = getCommandFormat().getOptValue(META_FLAG);
if (metaURI != null && !metaURI.isEmpty()) {
URI uri = URI.create(metaURI);
LOG.info("create metadata store: {}", uri + " scheme: "
+ uri.getScheme());
- switch (uri.getScheme().toLowerCase()) {
+ switch (uri.getScheme().toLowerCase(Locale.ENGLISH)) {
case "local":
- ms = new LocalMetadataStore();
+ setStore(new LocalMetadataStore());
break;
case "dynamodb":
- ms = new DynamoDBMetadataStore();
+ setStore(new DynamoDBMetadataStore());
conf.set(S3GUARD_DDB_TABLE_NAME_KEY, uri.getAuthority());
if (forceCreate) {
conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
@@ -213,19 +209,19 @@ public abstract class S3GuardTool extends Configured implements Tool {
} else {
// CLI does not specify metadata store URI, it uses default metadata store
// DynamoDB instead.
- ms = new DynamoDBMetadataStore();
+ setStore(new DynamoDBMetadataStore());
if (forceCreate) {
conf.setBoolean(S3GUARD_DDB_TABLE_CREATE_KEY, true);
}
}
- if (s3a == null) {
- ms.initialize(conf);
+ if (filesystem == null) {
+ getStore().initialize(conf);
} else {
- ms.initialize(s3a);
+ getStore().initialize(filesystem);
}
- LOG.info("Metadata store {} is initialized.", ms);
- return ms;
+ LOG.info("Metadata store {} is initialized.", getStore());
+ return getStore();
}
/**
@@ -252,7 +248,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
String.format("URI %s is not a S3A file system: %s", uri,
fs.getClass().getName()));
}
- s3a = (S3AFileSystem) fs;
+ filesystem = (S3AFileSystem) fs;
}
/**
@@ -263,7 +259,30 @@ public abstract class S3GuardTool extends Configured implements Tool {
* @return the position arguments from CLI.
*/
List<String> parseArgs(String[] args) {
- return commandFormat.parse(args, 1);
+ return getCommandFormat().parse(args, 1);
+ }
+
+ protected S3AFileSystem getFilesystem() {
+ return filesystem;
+ }
+
+ protected void setFilesystem(S3AFileSystem filesystem) {
+ this.filesystem = filesystem;
+ }
+
+ @VisibleForTesting
+ public MetadataStore getStore() {
+ return store;
+ }
+
+ @VisibleForTesting
+ protected void setStore(MetadataStore store) {
+ Preconditions.checkNotNull(store);
+ this.store = store;
+ }
+
+ protected CommandFormat getCommandFormat() {
+ return commandFormat;
}
/**
@@ -290,9 +309,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
Init(Configuration conf) {
super(conf);
// read capacity.
- commandFormat.addOptionWithValue(READ_FLAG);
+ getCommandFormat().addOptionWithValue(READ_FLAG);
// write capacity.
- commandFormat.addOptionWithValue(WRITE_FLAG);
+ getCommandFormat().addOptionWithValue(WRITE_FLAG);
}
@Override
@@ -309,12 +328,12 @@ public abstract class S3GuardTool extends Configured implements Tool {
public int run(String[] args) throws IOException {
List<String> paths = parseArgs(args);
- String readCap = commandFormat.getOptValue(READ_FLAG);
+ String readCap = getCommandFormat().getOptValue(READ_FLAG);
if (readCap != null && !readCap.isEmpty()) {
int readCapacity = Integer.parseInt(readCap);
getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_READ_KEY, readCapacity);
}
- String writeCap = commandFormat.getOptValue(WRITE_FLAG);
+ String writeCap = getCommandFormat().getOptValue(WRITE_FLAG);
if (writeCap != null && !writeCap.isEmpty()) {
int writeCapacity = Integer.parseInt(writeCap);
getConf().setInt(S3GUARD_DDB_TABLE_CAPACITY_WRITE_KEY, writeCapacity);
@@ -380,9 +399,10 @@ public abstract class S3GuardTool extends Configured implements Tool {
return SUCCESS;
}
- Preconditions.checkState(ms != null, "Metadata Store is not initialized");
+ Preconditions.checkState(getStore() != null,
+ "Metadata Store is not initialized");
- ms.destroy();
+ getStore().destroy();
LOG.info("Metadata store is deleted.");
return SUCCESS;
}
@@ -414,12 +434,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
super(conf);
}
- // temporary: for metadata store.
- @VisibleForTesting
- void setMetadataStore(MetadataStore ms) {
- this.ms = ms;
- }
-
@Override
String getName() {
return NAME;
@@ -445,7 +459,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
FileStatus dir = DynamoDBMetadataStore.makeDirStatus(parent,
f.getOwner());
- ms.put(new PathMetadata(dir));
+ getStore().put(new PathMetadata(dir));
dirCache.add(parent);
parent = parent.getParent();
}
@@ -458,8 +472,8 @@ public abstract class S3GuardTool extends Configured implements Tool {
*/
private long importDir(FileStatus status) throws IOException {
Preconditions.checkArgument(status.isDirectory());
- RemoteIterator<LocatedFileStatus> it =
- s3a.listFilesAndEmptyDirectories(status.getPath(), true);
+ RemoteIterator<LocatedFileStatus> it = getFilesystem()
+ .listFilesAndEmptyDirectories(status.getPath(), true);
long items = 0;
while (it.hasNext()) {
@@ -477,7 +491,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
located.getOwner());
}
putParentsIfNotPresent(child);
- ms.put(new PathMetadata(child));
+ getStore().put(new PathMetadata(child));
items++;
}
return items;
@@ -506,14 +520,14 @@ public abstract class S3GuardTool extends Configured implements Tool {
filePath = "/";
}
Path path = new Path(filePath);
- FileStatus status = s3a.getFileStatus(path);
+ FileStatus status = getFilesystem().getFileStatus(path);
initMetadataStore(false);
long items = 1;
if (status.isFile()) {
PathMetadata meta = new PathMetadata(status);
- ms.put(meta);
+ getStore().put(meta);
} else {
items = importDir(status);
}
@@ -552,12 +566,6 @@ public abstract class S3GuardTool extends Configured implements Tool {
super(conf);
}
- @VisibleForTesting
- void setMetadataStore(MetadataStore ms) {
- Preconditions.checkNotNull(ms);
- this.ms = ms;
- }
-
@Override
String getName() {
return NAME;
@@ -651,7 +659,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
Map<Path, FileStatus> s3Children = new HashMap<>();
if (s3Dir != null && s3Dir.isDirectory()) {
- for (FileStatus status : s3a.listStatus(s3Dir.getPath())) {
+ for (FileStatus status : getFilesystem().listStatus(s3Dir.getPath())) {
s3Children.put(status.getPath(), status);
}
}
@@ -659,7 +667,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
Map<Path, FileStatus> msChildren = new HashMap<>();
if (msDir != null && msDir.isDirectory()) {
DirListingMetadata dirMeta =
- ms.listChildren(msDir.getPath());
+ getStore().listChildren(msDir.getPath());
if (dirMeta != null) {
for (PathMetadata meta : dirMeta.getListing()) {
@@ -692,13 +700,13 @@ public abstract class S3GuardTool extends Configured implements Tool {
* @throws IOException on I/O errors.
*/
private void compareRoot(Path path, PrintStream out) throws IOException {
- Path qualified = s3a.qualify(path);
+ Path qualified = getFilesystem().qualify(path);
FileStatus s3Status = null;
try {
- s3Status = s3a.getFileStatus(qualified);
+ s3Status = getFilesystem().getFileStatus(qualified);
} catch (FileNotFoundException e) {
}
- PathMetadata meta = ms.get(qualified);
+ PathMetadata meta = getStore().get(qualified);
FileStatus msStatus = (meta != null && !meta.isDeleted()) ?
meta.getFileStatus() : null;
compareDir(msStatus, s3Status, out);
@@ -727,7 +735,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
} else {
root = new Path(uri.getPath());
}
- root = s3a.qualify(root);
+ root = getFilesystem().qualify(root);
compareRoot(root, out);
out.flush();
return SUCCESS;
@@ -763,16 +771,17 @@ public abstract class S3GuardTool extends Configured implements Tool {
Prune(Configuration conf) {
super(conf);
- commandFormat.addOptionWithValue(DAYS_FLAG);
- commandFormat.addOptionWithValue(HOURS_FLAG);
- commandFormat.addOptionWithValue(MINUTES_FLAG);
- commandFormat.addOptionWithValue(SECONDS_FLAG);
+ CommandFormat format = getCommandFormat();
+ format.addOptionWithValue(DAYS_FLAG);
+ format.addOptionWithValue(HOURS_FLAG);
+ format.addOptionWithValue(MINUTES_FLAG);
+ format.addOptionWithValue(SECONDS_FLAG);
}
@VisibleForTesting
void setMetadataStore(MetadataStore ms) {
Preconditions.checkNotNull(ms);
- this.ms = ms;
+ this.setStore(ms);
}
@Override
@@ -786,7 +795,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
private long getDeltaComponent(TimeUnit unit, String arg) {
- String raw = commandFormat.getOptValue(arg);
+ String raw = getCommandFormat().getOptValue(arg);
if (raw == null || raw.isEmpty()) {
return 0;
}
@@ -827,7 +836,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
long now = System.currentTimeMillis();
long divide = now - delta;
- ms.prune(divide);
+ getStore().prune(divide);
out.flush();
return SUCCESS;
@@ -839,15 +848,15 @@ public abstract class S3GuardTool extends Configured implements Tool {
}
}
- private static S3GuardTool cmd;
+ private static S3GuardTool command;
private static void printHelp() {
- if (cmd == null) {
+ if (command == null) {
System.err.println("Usage: hadoop " + USAGE);
System.err.println("\tperform S3Guard metadata store " +
"administrative commands.");
} else {
- System.err.println("Usage: hadoop " + cmd.getUsage());
+ System.err.println("Usage: hadoop " + command.getUsage());
}
System.err.println();
System.err.println(COMMON_USAGE);
@@ -874,25 +883,25 @@ public abstract class S3GuardTool extends Configured implements Tool {
final String subCommand = otherArgs[0];
switch (subCommand) {
case Init.NAME:
- cmd = new Init(conf);
+ command = new Init(conf);
break;
case Destroy.NAME:
- cmd = new Destroy(conf);
+ command = new Destroy(conf);
break;
case Import.NAME:
- cmd = new Import(conf);
+ command = new Import(conf);
break;
case Diff.NAME:
- cmd = new Diff(conf);
+ command = new Diff(conf);
break;
case Prune.NAME:
- cmd = new Prune(conf);
+ command = new Prune(conf);
break;
default:
printHelp();
return INVALID_ARGUMENT;
}
- return ToolRunner.run(conf, cmd, otherArgs);
+ return ToolRunner.run(conf, command, otherArgs);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
index 60bfa23..7abd474 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ADelayedFNF.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Test;
import java.io.FileNotFoundException;
+import java.util.concurrent.Callable;
/**
* Tests behavior of a FileNotFound error that happens after open(), i.e. on
@@ -45,11 +46,17 @@ public class ITestS3ADelayedFNF extends AbstractS3ATestBase {
Path p = path("some-file");
ContractTestUtils.createFile(fs, p, false, new byte[] {20, 21, 22});
- FSDataInputStream in = fs.open(p);
+ final FSDataInputStream in = fs.open(p);
assertDeleted(p, false);
// This should fail since we deleted after the open.
- LambdaTestUtils.intercept(FileNotFoundException.class, () -> in.read());
+ LambdaTestUtils.intercept(FileNotFoundException.class,
+ new Callable<Integer>() {
+ @Override
+ public Integer call() throws Exception {
+ return in.read();
+ }
+ });
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
index 2d47bb3..27af23a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileSystemContract.java
@@ -27,11 +27,10 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystemContractBaseTest;
import org.apache.hadoop.fs.Path;
+
import static org.junit.Assume.*;
import static org.junit.Assert.*;
-import java.io.IOException;
-
/**
* Tests a live S3 system. If your keys and bucket aren't specified, all tests
* are marked as passed.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
index dd20903..869d64c 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AMiscOperations.java
@@ -26,11 +26,13 @@ import org.apache.hadoop.test.LambdaTestUtils;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectResult;
import org.junit.Test;
import java.io.ByteArrayInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.concurrent.Callable;
/**
* Tests of the S3A FileSystem which don't have a specific home and can share
@@ -62,7 +64,7 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
@Test
public void testPutObjectDirect() throws Throwable {
- S3AFileSystem fs = getFileSystem();
+ final S3AFileSystem fs = getFileSystem();
ObjectMetadata metadata = fs.newObjectMetadata(-1);
metadata.setContentLength(-1);
Path path = path("putDirect");
@@ -71,7 +73,12 @@ public class ITestS3AMiscOperations extends AbstractS3ATestBase {
new ByteArrayInputStream("PUT".getBytes()),
metadata);
LambdaTestUtils.intercept(IllegalStateException.class,
- () -> fs.putObjectDirect(put));
+ new Callable<PutObjectResult>() {
+ @Override
+ public PutObjectResult call() throws Exception {
+ return fs.putObjectDirect(put);
+ }
+ });
assertPathDoesNotExist("put object was created", path);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index 5f34795..ceacdf3 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
index d584850..0291acd 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBLocalClientFactory.java
@@ -124,13 +124,14 @@ public class DynamoDBLocalClientFactory extends Configured
try {
// Start an in-memory local DynamoDB instance
final String port = String.valueOf(ServerSocketUtil.getPort(0, 100));
+ ddbEndpoint = "http://localhost:" + port;
dynamoDBLocalServer = ServerRunner.createServerFromCommandLineArgs(
new String[]{"-inMemory", "-port", port});
dynamoDBLocalServer.start();
- ddbEndpoint = "http://localhost:" + port;
LOG.info("DynamoDBLocal singleton server was started at {}", ddbEndpoint);
- } catch (Throwable t) {
- String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint;
+ } catch (Exception t) {
+ String msg = "Error starting DynamoDBLocal server at " + ddbEndpoint
+ + " " + t;
LOG.error(msg, t);
throw new IOException(msg, t);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
index 21f2cc8..c6838a0 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
@@ -80,7 +80,7 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
@Test
public void testConcurrentTableCreations() throws Exception {
- Configuration conf = getConfiguration();
+ final Configuration conf = getConfiguration();
Assume.assumeTrue("Test only applies when DynamoDB is used for S3Guard",
conf.get(Constants.S3_METADATA_STORE_IMPL).equals(
Constants.S3GUARD_METASTORE_DYNAMO));
@@ -121,7 +121,8 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
@Override
public Exception call() throws Exception {
- ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ ContractTestUtils.NanoTimer timer =
+ new ContractTestUtils.NanoTimer();
Exception result = null;
try (DynamoDBMetadataStore store = new DynamoDBMetadataStore()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
index d2aa56f..c13dfc4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolDynamoDB.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -59,9 +59,9 @@ public class ITestS3GuardToolDynamoDB extends AbstractS3GuardToolTestBase {
@Test
public void testInvalidRegion() throws Exception {
final String testTableName = "testInvalidRegion" + new Random().nextInt();
- String testRegion = "invalidRegion";
+ final String testRegion = "invalidRegion";
// Initialize MetadataStore
- Init initCmd = new Init(getFileSystem().getConf());
+ final Init initCmd = new Init(getFileSystem().getConf());
LambdaTestUtils.intercept(IOException.class,
new Callable<String>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
index 992b8f6..181cdfb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardToolLocal.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.Diff;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
-
/**
* Test S3Guard related CLI commands against a LocalMetadataStore.
*/
@@ -65,7 +64,7 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
}
S3GuardTool.Import cmd = new S3GuardTool.Import(fs.getConf());
- cmd.setMetadataStore(ms);
+ cmd.setStore(ms);
expectSuccess("Import command did not exit successfully - see output",
cmd,
@@ -111,7 +110,7 @@ public class ITestS3GuardToolLocal extends AbstractS3GuardToolTestBase {
ByteArrayOutputStream buf = new ByteArrayOutputStream();
PrintStream out = new PrintStream(buf);
Diff cmd = new Diff(fs.getConf());
- cmd.setMetadataStore(ms);
+ cmd.setStore(ms);
assertEquals("Diff command did not exit successfully - see output", SUCCESS,
cmd.run(new String[]{"diff", "-meta", "local://metadata",
testPath.toString()}, out));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
index 957ebe0..8458252 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDirListingMetadata.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -18,8 +18,8 @@
package org.apache.hadoop.fs.s3a.s3guard;
+import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.List;
import org.junit.Rule;
@@ -63,7 +63,8 @@ public class TestDirListingMetadata {
public void testEmptyListing() {
Path path = new Path("/path");
DirListingMetadata meta = new DirListingMetadata(path,
- Collections.emptyList(), false);
+ new ArrayList<PathMetadata>(0),
+ false);
assertEquals(path, meta.getPath());
assertNotNull(meta.getListing());
assertTrue(meta.getListing().isEmpty());
@@ -294,7 +295,8 @@ public class TestDirListingMetadata {
PathMetadata pathMeta2 = new PathMetadata(
new S3AFileStatus(true, new Path(parent, "dir2"), TEST_OWNER));
PathMetadata pathMeta3 = new PathMetadata(
- new S3AFileStatus(123, 456, new Path(parent, "file1"), 8192, TEST_OWNER));
+ new S3AFileStatus(123, 456, new Path(parent, "file1"), 8192,
+ TEST_OWNER));
List<PathMetadata> listing = Arrays.asList(pathMeta1, pathMeta2, pathMeta3);
return new DirListingMetadata(parent, listing, false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index 038a399..02eb7b8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
@@ -105,7 +106,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
+ "server is not configured correctly. ";
LOG.error(msg, e);
// fail fast if the DynamoDBLocal server can not work
- fail(msg + e.getMessage());
+ throw e;
}
}
@@ -274,14 +275,14 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
ms.put(new PathMetadata(basicFileStatus(oldDir, 0, true)));
ms.put(new PathMetadata(basicFileStatus(newDir, 0, true)));
- final Collection<PathMetadata> oldMetas =
- numDelete < 0 ? null : new ArrayList<>(numDelete);
+ final List<PathMetadata> oldMetas =
+ numDelete < 0 ? null : new ArrayList<PathMetadata>(numDelete);
for (int i = 0; i < numDelete; i++) {
oldMetas.add(new PathMetadata(
basicFileStatus(new Path(oldDir, "child" + i), i, true)));
}
- final Collection<PathMetadata> newMetas =
- numPut < 0 ? null : new ArrayList<>(numPut);
+ final List<PathMetadata> newMetas =
+ numPut < 0 ? null : new ArrayList<PathMetadata>(numPut);
for (int i = 0; i < numPut; i++) {
newMetas.add(new PathMetadata(
basicFileStatus(new Path(newDir, "child" + i), i, false)));
@@ -336,9 +337,14 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
@Test
public void testItemLacksVersion() throws Throwable {
intercept(IOException.class, E_NOT_VERSION_MARKER,
- () -> verifyVersionCompatibility("table",
- new Item().withPrimaryKey(
- createVersionMarkerPrimaryKey(VERSION_MARKER))));
+ new VoidCallable() {
+ @Override
+ public void call() throws Exception {
+ verifyVersionCompatibility("table",
+ new Item().withPrimaryKey(
+ createVersionMarkerPrimaryKey(VERSION_MARKER)));
+ }
+ });
}
/**
@@ -358,7 +364,12 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
// create existing table
intercept(IOException.class, E_NO_VERSION_MARKER,
- () -> ddbms.initTable());
+ new VoidCallable() {
+ @Override
+ public void call() throws Exception {
+ ddbms.initTable();
+ }
+ });
conf.setInt(S3GUARD_DDB_MAX_RETRIES, maxRetries);
}
@@ -378,7 +389,13 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
table.putItem(v200);
// create existing table
- intercept(IOException.class, E_INCOMPATIBLE_VERSION, ddbms::initTable);
+ intercept(IOException.class, E_INCOMPATIBLE_VERSION,
+ new VoidCallable() {
+ @Override
+ public void call() throws Exception {
+ ddbms.initTable();
+ }
+ });
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f07c7aa3/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
index ffd64ef..1678746 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestPathMetadataDynamoDBTranslation.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.fs.s3a.s3guard;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
+import java.util.concurrent.Callable;
import com.amazonaws.services.dynamodbv2.document.Item;
import com.amazonaws.services.dynamodbv2.document.KeyAttribute;
@@ -193,7 +194,12 @@ public class TestPathMetadataDynamoDBTranslation extends Assert {
@Test
public void testPathToKey() throws Exception {
LambdaTestUtils.intercept(IllegalArgumentException.class,
- () -> pathToKey(new Path("/")));
+ new Callable<PrimaryKey>() {
+ @Override
+ public PrimaryKey call() throws Exception {
+ return pathToKey(new Path("/"));
+ }
+ });
doTestPathToKey(TEST_DIR_PATH);
doTestPathToKey(TEST_FILE_PATH);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org