You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ma...@apache.org on 2017/04/07 23:49:51 UTC
hadoop git commit: HADOOP-14288. TestDynamoDBMEtadataStore is broken
unless we can fail faster without a table version
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13345 26e1eaa60 -> 13fafee1b
HADOOP-14288. TestDynamoDBMEtadataStore is broken unless we can fail faster without a table version
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13fafee1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13fafee1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13fafee1
Branch: refs/heads/HADOOP-13345
Commit: 13fafee1be950cda8cef96d089e5ee4be223bba0
Parents: 26e1eaa
Author: Sean Mackrory <ma...@apache.org>
Authored: Fri Apr 7 17:48:42 2017 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Fri Apr 7 17:48:50 2017 -0600
----------------------------------------------------------------------
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 9 ++++----
.../fs/s3a/s3guard/MetadataStoreTestBase.java | 11 +++++++++-
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 23 ++++++++++++++++----
.../fs/s3a/s3guard/TestLocalMetadataStore.java | 11 +++++++++-
.../fs/s3a/s3guard/TestNullMetadataStore.java | 5 +++++
5 files changed, 49 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index e0a171d..2b28c58 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -191,7 +191,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
private Configuration conf;
private String username;
- private RetryPolicy batchRetryPolicy;
+ private RetryPolicy dataAccessRetryPolicy;
/**
* A utility function to create DynamoDB instance.
@@ -274,7 +274,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
private void setMaxRetries(Configuration config) {
int maxRetries = config.getInt(S3GUARD_DDB_MAX_RETRIES,
S3GUARD_DDB_MAX_RETRIES_DEFAULT);
- batchRetryPolicy = RetryPolicies
+ dataAccessRetryPolicy = RetryPolicies
.exponentialBackoffRetry(maxRetries, MIN_RETRY_SLEEP_MSEC,
TimeUnit.MILLISECONDS);
}
@@ -517,7 +517,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
private void retryBackoff(int retryCount) throws IOException {
try {
// Our RetryPolicy ignores everything but retryCount here.
- RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null,
+ RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null,
retryCount, 0, true);
if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
throw new IOException(
@@ -764,7 +764,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
Item versionMarker = table.getItem(versionMarkerKey);
while (versionMarker == null) {
try {
- RetryPolicy.RetryAction action = batchRetryPolicy.shouldRetry(null,
+ RetryPolicy.RetryAction action = dataAccessRetryPolicy.shouldRetry(null,
retryCount, 0, true);
if (action.action == RetryPolicy.RetryAction.RetryDecision.FAIL) {
break;
@@ -775,6 +775,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
} catch (Exception e) {
throw new IOException("initTable: Unexpected exception", e);
}
+ retryCount++;
versionMarker = table.getItem(versionMarkerKey);
}
return versionMarker;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
index 36a5cc5..88df45e 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/MetadataStoreTestBase.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3a.s3guard;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -63,12 +64,20 @@ public abstract class MetadataStoreTestBase extends Assert {
private final long modTime = accessTime - 5000;
/**
- * Each test should override this.
+ * Each test should override this. Will use a new Configuration instance.
* @return Contract which specifies the MetadataStore under test plus config.
*/
public abstract AbstractMSContract createContract() throws IOException;
/**
+ * Each test should override this.
+ * @param conf Base configuration instance to use.
+ * @return Contract which specifies the MetadataStore under test plus config.
+ */
+ public abstract AbstractMSContract createContract(Configuration conf)
+ throws IOException;
+
+ /**
* Tests assume that implementations will return recently set results. If
* your implementation does not always hold onto metadata (e.g. LRU or
* time-based expiry) you can override this to return false.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index a841e21..3584b54 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -126,7 +126,10 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
private final DynamoDBMetadataStore ms = new DynamoDBMetadataStore();
DynamoDBMSContract() throws IOException {
- final Configuration conf = new Configuration();
+ this(new Configuration());
+ }
+
+ DynamoDBMSContract(Configuration conf) throws IOException {
// using mocked S3 clients
conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
S3ClientFactory.class);
@@ -160,6 +163,12 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
}
@Override
+ public DynamoDBMSContract createContract(Configuration conf) throws
+ IOException {
+ return new DynamoDBMSContract(conf);
+ }
+
+ @Override
FileStatus basicFileStatus(Path path, int size, boolean isDir)
throws IOException {
String owner = UserGroupInformation.getCurrentUser().getShortUserName();
@@ -340,15 +349,21 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
*/
@Test
public void testTableVersionRequired() throws Exception {
- final DynamoDBMetadataStore ddbms = createContract().getMetadataStore();
- String tableName = getFileSystem().getConf().get(Constants
- .S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
+ Configuration conf = getFileSystem().getConf();
+ int maxRetries = conf.getInt(Constants.S3GUARD_DDB_MAX_RETRIES, Constants
+ .S3GUARD_DDB_MAX_RETRIES_DEFAULT);
+ conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, 3);
+
+ final DynamoDBMetadataStore ddbms = createContract(conf).getMetadataStore();
+ String tableName = conf.get(Constants.S3GUARD_DDB_TABLE_NAME_KEY, BUCKET);
Table table = verifyTableInitialized(tableName);
table.deleteItem(VERSION_MARKER_PRIMARY_KEY);
// create existing table
intercept(IOException.class, E_NO_VERSION_MARKER,
() -> ddbms.initTable());
+
+ conf.setInt(Constants.S3GUARD_DDB_MAX_RETRIES, maxRetries);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
index c3eb0d3..4cffc6f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestLocalMetadataStore.java
@@ -42,7 +42,10 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
private FileSystem fs;
private LocalMSContract() throws IOException {
- Configuration config = new Configuration();
+ this(new Configuration());
+ }
+
+ private LocalMSContract(Configuration config) throws IOException {
config.set(LocalMetadataStore.CONF_MAX_RECORDS, MAX_ENTRIES_STR);
fs = FileSystem.getLocal(config);
}
@@ -64,6 +67,12 @@ public class TestLocalMetadataStore extends MetadataStoreTestBase {
return new LocalMSContract();
}
+ @Override
+ public AbstractMSContract createContract(Configuration conf) throws
+ IOException {
+ return new LocalMSContract(conf);
+ }
+
@Test
public void testClearByAncestor() {
Map<Path, String> map = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13fafee1/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
index 5b18686..5b19efa 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestNullMetadataStore.java
@@ -55,4 +55,9 @@ public class TestNullMetadataStore extends MetadataStoreTestBase {
public AbstractMSContract createContract() {
return new NullMSContract();
}
+
+ @Override
+ public AbstractMSContract createContract(Configuration conf) {
+ return createContract();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org