You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/08/09 11:05:47 UTC
hadoop git commit: HADOOP-14733. ITestS3GuardConcurrentOps failing
with -Ddynamodblocal -Ds3guard. Contributed by Steve Loughran.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 b8ca3bdf3 -> 7bd0284b9
HADOOP-14733. ITestS3GuardConcurrentOps failing with -Ddynamodblocal -Ds3guard.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7bd0284b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7bd0284b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7bd0284b
Branch: refs/heads/HADOOP-13345
Commit: 7bd0284b951649de91c981717d24e4e141f42bf4
Parents: b8ca3bd
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 9 12:04:30 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 9 12:04:30 2017 +0100
----------------------------------------------------------------------
.../s3a/s3guard/ITestS3GuardConcurrentOps.java | 55 ++++++++++++++------
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 6 +--
2 files changed, 40 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd0284b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
index d90d2db..24eb6fb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/ITestS3GuardConcurrentOps.java
@@ -22,6 +22,8 @@ import com.amazonaws.services.dynamodbv2.document.DynamoDB;
import com.amazonaws.services.dynamodbv2.document.Table;
import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
@@ -36,11 +38,14 @@ import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.commons.lang3.StringUtils;
import org.junit.Assume;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
+import static org.apache.hadoop.fs.s3a.Constants.S3GUARD_DDB_REGION_KEY;
+
/**
* Tests concurrent operations on S3Guard.
*/
@@ -64,10 +69,14 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
private void deleteTable(DynamoDB db, String tableName) throws
InterruptedException {
- Table table = db.getTable(tableName);
- table.waitForActive();
- table.delete();
- table.waitForDelete();
+ try {
+ Table table = db.getTable(tableName);
+ table.waitForActive();
+ table.delete();
+ table.waitForDelete();
+ } catch (ResourceNotFoundException e) {
+ LOG.warn("Failed to delete {}, as it was not found", tableName, e);
+ }
}
@Test
@@ -84,6 +93,12 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
String tableName = "testConcurrentTableCreations" + new Random().nextInt();
conf.setBoolean(Constants.S3GUARD_DDB_TABLE_CREATE_KEY, true);
conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName);
+
+ String region = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
+ if (StringUtils.isEmpty(region)) {
+ // no region set, so pick it up from the test bucket
+ conf.set(S3GUARD_DDB_REGION_KEY, getFileSystem().getBucketLocation());
+ }
int concurrentOps = 16;
int iterations = 4;
@@ -100,39 +115,45 @@ public class ITestS3GuardConcurrentOps extends AbstractS3ATestBase {
}
});
((ThreadPoolExecutor) executor).prestartAllCoreThreads();
- Future<Boolean>[] futures = new Future[concurrentOps];
- int exceptionsThrown = 0;
+ Future<Exception>[] futures = new Future[concurrentOps];
for (int f = 0; f < concurrentOps; f++) {
final int index = f;
- futures[f] = executor.submit(new Callable<Boolean>() {
+ futures[f] = executor.submit(new Callable<Exception>() {
@Override
- public Boolean call() throws Exception {
+ public Exception call() throws Exception {
+
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
- boolean result = false;
- try {
- new DynamoDBMetadataStore().initialize(conf);
+ Exception result = null;
+ try (DynamoDBMetadataStore store = new DynamoDBMetadataStore()) {
+ store.initialize(conf);
} catch (Exception e) {
LOG.error(e.getClass() + ": " + e.getMessage());
- result = true;
+ result = e;
}
- timer.end("parallel DynamoDB client creation %d", index);
+ timer.end("Parallel DynamoDB client creation %d", index);
LOG.info("Parallel DynamoDB client creation {} ran from {} to {}",
index, timer.getStartTime(), timer.getEndTime());
return result;
}
});
}
+ List<Exception> exceptions = new ArrayList<>(concurrentOps);
for (int f = 0; f < concurrentOps; f++) {
- if (futures[f].get()) {
- exceptionsThrown++;
+ Exception outcome = futures[f].get();
+ if (outcome != null) {
+ exceptions.add(outcome);
}
}
deleteTable(db, tableName);
+ int exceptionsThrown = exceptions.size();
if (exceptionsThrown > 0) {
- fail(exceptionsThrown + "/" + concurrentOps +
- " threads threw exceptions while initializing on iteration " + i);
+ // at least one exception was thrown. Fail the test & nest the first
+ // exception caught
+ throw new AssertionError(exceptionsThrown + "/" + concurrentOps +
+ " threads threw exceptions while initializing on iteration " + i,
+ exceptions.get(0));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7bd0284b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index 27416bb..bde624d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -221,16 +221,14 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
String savedRegion = conf.get(Constants.S3GUARD_DDB_REGION_KEY,
getFileSystem().getBucketLocation());
conf.unset(Constants.S3GUARD_DDB_REGION_KEY);
- try {
- DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
+ try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()) {
ddbms.initialize(conf);
fail("Should have failed because the table name is not set!");
} catch (IllegalArgumentException ignored) {
}
// config table name
conf.set(Constants.S3GUARD_DDB_TABLE_NAME_KEY, tableName);
- try {
- DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore();
+ try (DynamoDBMetadataStore ddbms = new DynamoDBMetadataStore()){
ddbms.initialize(conf);
fail("Should have failed because as the region is not set!");
} catch (IllegalArgumentException ignored) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org