You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2018/10/12 12:28:41 UTC
[24/50] [abbrv] hadoop git commit: HADOOP-15837. DynamoDB table
Update can fail S3A FS init. Contributed by Steve Loughran.
HADOOP-15837. DynamoDB table Update can fail S3A FS init.
Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee816f1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee816f1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee816f1f
Branch: refs/heads/HDFS-13891
Commit: ee816f1fd78b029b9efa567e8b1b62391563de14
Parents: 8a37983
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 11 14:57:38 2018 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 11 14:57:38 2018 +0100
----------------------------------------------------------------------
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 89 +++++++++++++++---
.../s3a/s3guard/TestDynamoDBMiscOperations.java | 94 ++++++++++++++++++++
2 files changed, 169 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee816f1f/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index f34afb5..6f892e5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -42,6 +42,7 @@ import java.util.stream.Collectors;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkBaseException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
import com.amazonaws.services.dynamodbv2.document.BatchWriteItemOutcome;
@@ -66,6 +67,7 @@ import com.amazonaws.services.dynamodbv2.model.TableDescription;
import com.amazonaws.services.dynamodbv2.model.Tag;
import com.amazonaws.services.dynamodbv2.model.TagResourceRequest;
import com.amazonaws.services.dynamodbv2.model.WriteRequest;
+import com.amazonaws.waiters.WaiterTimedOutException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import org.slf4j.Logger;
@@ -78,6 +80,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
import org.apache.hadoop.fs.s3a.AWSCredentialProviderList;
import org.apache.hadoop.fs.s3a.AWSServiceThrottledException;
import org.apache.hadoop.fs.s3a.Constants;
@@ -1129,6 +1132,9 @@ public class DynamoDBMetadataStore implements MetadataStore {
* overall, this method is synchronous, and the table is guaranteed to exist
* after this method returns successfully.
*
+ * The wait for a table becoming active is Retry+Translated; it can fail
+ * while a table is not yet ready.
+ *
* @throws IOException if table does not exist and auto-creation is disabled;
* or table is being deleted, or any other I/O exception occurred.
*/
@@ -1144,7 +1150,6 @@ public class DynamoDBMetadataStore implements MetadataStore {
final String status = description.getTableStatus();
switch (status) {
case "CREATING":
- case "UPDATING":
LOG.debug("Table {} in region {} is being created/updated. This may"
+ " indicate that the table is being operated by another "
+ "concurrent thread or process. Waiting for active...",
@@ -1155,6 +1160,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
throw new FileNotFoundException("DynamoDB table "
+ "'" + tableName + "' is being "
+ "deleted in region " + region);
+ case "UPDATING":
+ // table being updated; it can still be used.
+ LOG.debug("Table is being updated.");
+ break;
case "ACTIVE":
break;
default:
@@ -1285,24 +1294,34 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @throws InterruptedIOException if the wait was interrupted
* @throws IllegalArgumentException if an exception was raised in the waiter
*/
- @Retries.OnceRaw
- private void waitForTableActive(Table t) throws InterruptedIOException {
- try {
- t.waitForActive();
- } catch (InterruptedException e) {
- LOG.warn("Interrupted while waiting for table {} in region {} active",
- tableName, region, e);
- Thread.currentThread().interrupt();
- throw (InterruptedIOException)
- new InterruptedIOException("DynamoDB table '"
- + tableName + "' is not active yet in region " + region)
- .initCause(e);
- }
+ @Retries.RetryTranslated
+ private void waitForTableActive(Table t) throws IOException {
+ invoker.retry("Waiting for active state of table " + tableName,
+ null,
+ true,
+ () -> {
+ try {
+ t.waitForActive();
+ } catch (IllegalArgumentException ex) {
+ throw translateTableWaitFailure(tableName, ex);
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted while waiting for table {} in region {}"
+ + " active",
+ tableName, region, e);
+ Thread.currentThread().interrupt();
+ throw (InterruptedIOException)
+ new InterruptedIOException("DynamoDB table '"
+ + tableName + "' is not active yet in region " + region)
+ .initCause(e);
+ }
+ });
}
/**
* Create a table, wait for it to become active, then add the version
* marker.
+ * Creating an setting up the table isn't wrapped by any retry operations;
+ * the wait for a table to become available is RetryTranslated.
* @param capacity capacity to provision
* @throws IOException on any failure.
* @throws InterruptedIOException if the wait was interrupted
@@ -1604,4 +1623,46 @@ public class DynamoDBMetadataStore implements MetadataStore {
public Invoker getInvoker() {
return invoker;
}
+
+ /**
+ * Take an {@code IllegalArgumentException} raised by a DDB operation
+ * and if it contains an inner SDK exception, unwrap it.
+ * @param ex exception.
+ * @return the inner AWS exception or null.
+ */
+ public static SdkBaseException extractInnerException(
+ IllegalArgumentException ex) {
+ if (ex.getCause() instanceof SdkBaseException) {
+ return (SdkBaseException) ex.getCause();
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Handle a table wait failure by extracting any inner cause and
+ * converting it, or, if unconvertable by wrapping
+ * the IllegalArgumentException in an IOE.
+ *
+ * @param name name of the table
+ * @param e exception
+ * @return an IOE to raise.
+ */
+ @VisibleForTesting
+ static IOException translateTableWaitFailure(
+ final String name, IllegalArgumentException e) {
+ final SdkBaseException ex = extractInnerException(e);
+ if (ex != null) {
+ if (ex instanceof WaiterTimedOutException) {
+ // a timeout waiting for state change: extract the
+ // message from the outer exception, but translate
+ // the inner one for the throttle policy.
+ return new AWSClientIOException(e.getMessage(), ex);
+ } else {
+ return translateException(e.getMessage(), name, ex);
+ }
+ } else {
+ return new IOException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee816f1f/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
new file mode 100644
index 0000000..03be39f
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMiscOperations.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.s3guard;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
+import com.amazonaws.waiters.WaiterTimedOutException;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.s3guard.DynamoDBMetadataStore.translateTableWaitFailure;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Unit test suite for misc dynamoDB metastore operations.
+ */
+public class TestDynamoDBMiscOperations extends HadoopTestBase {
+
+ private static final String TIMEOUT_ERROR_MESSAGE
+ = "Table table-name did not transition into ACTIVE state.";
+
+ @Test
+ public void testUnwrapTableWaitTimeout() throws Throwable {
+ final Exception waiterTimedOut =
+ new WaiterTimedOutException("waiter timed out");
+ final AWSClientIOException ex = intercept(AWSClientIOException.class,
+ TIMEOUT_ERROR_MESSAGE,
+ () -> {
+ throw translateTableWaitFailure("example",
+ new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE,
+ waiterTimedOut));
+ });
+ assertEquals(waiterTimedOut, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateIllegalArgumentException() throws Throwable {
+ final IllegalArgumentException e =
+ new IllegalArgumentException(TIMEOUT_ERROR_MESSAGE);
+ final IOException ex = intercept(IOException.class,
+ TIMEOUT_ERROR_MESSAGE,
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(e, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateWrappedDDBException() throws Throwable {
+ final Exception inner = new ResourceNotFoundException("ddb");
+ final IllegalArgumentException e =
+ new IllegalArgumentException("outer", inner);
+ final FileNotFoundException ex = intercept(FileNotFoundException.class,
+ "outer",
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(inner, ex.getCause());
+ }
+
+ @Test
+ public void testTranslateWrappedOtherException() throws Throwable {
+ final Exception inner = new NullPointerException("npe");
+ final IllegalArgumentException e =
+ new IllegalArgumentException("outer", inner);
+ final IOException ex = intercept(IOException.class,
+ "outer",
+ () -> {
+ throw translateTableWaitFailure("example", e);
+ });
+ assertEquals(e, ex.getCause());
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org