You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/11/03 22:09:21 UTC
[05/50] hadoop git commit: YARN-4288. Fixed RMProxy to retry on
IOException from local host. Contributed by Junping Du
YARN-4288. Fixed RMProxy to retry on IOException from local host. Contributed by Junping Du
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4169996
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4169996
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4169996
Branch: refs/heads/HDFS-8966
Commit: c41699965e78ce5e87669d17923ab84e494c4188
Parents: 5eca6de
Author: Jian He <ji...@apache.org>
Authored: Thu Oct 29 00:00:48 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Oct 29 00:00:48 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/io/retry/RetryPolicies.java | 44 +++++++++++++++++++-
.../apache/hadoop/io/retry/TestRetryProxy.java | 27 +++++++++++-
.../io/retry/UnreliableImplementation.java | 17 ++++++++
.../hadoop/io/retry/UnreliableInterface.java | 3 ++
hadoop-yarn-project/CHANGES.txt | 3 ++
.../org/apache/hadoop/yarn/client/RMProxy.java | 6 ++-
6 files changed, 94 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index f978ae7..171d52a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -139,7 +139,17 @@ public class RetryPolicies {
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
return new RemoteExceptionDependentRetry(defaultPolicy, exceptionToPolicyMap);
}
-
+
+ /**
+ * A retry policy for exceptions other than RemoteException.
+ */
+ public static final RetryPolicy retryOtherThanRemoteException(
+ RetryPolicy defaultPolicy,
+ Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap) {
+ return new OtherThanRemoteExceptionDependentRetry(defaultPolicy,
+ exceptionToPolicyMap);
+ }
+
public static final RetryPolicy failoverOnNetworkException(int maxFailovers) {
return failoverOnNetworkException(TRY_ONCE_THEN_FAIL, maxFailovers);
}
@@ -489,7 +499,37 @@ public class RetryPolicies {
return policy.shouldRetry(e, retries, failovers, isIdempotentOrAtMostOnce);
}
}
-
+
+ static class OtherThanRemoteExceptionDependentRetry implements RetryPolicy {
+
+ private RetryPolicy defaultPolicy;
+ private Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap;
+
+ public OtherThanRemoteExceptionDependentRetry(RetryPolicy defaultPolicy,
+ Map<Class<? extends Exception>,
+ RetryPolicy> exceptionToPolicyMap) {
+ this.defaultPolicy = defaultPolicy;
+ this.exceptionToPolicyMap = exceptionToPolicyMap;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e, int retries, int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ RetryPolicy policy = null;
+ // ignore Remote Exception
+ if (e instanceof RemoteException) {
+ // do nothing
+ } else {
+ policy = exceptionToPolicyMap.get(e.getClass());
+ }
+ if (policy == null) {
+ policy = defaultPolicy;
+ }
+ return policy.shouldRetry(
+ e, retries, failovers, isIdempotentOrAtMostOnce);
+ }
+ }
+
static class ExponentialBackoffRetry extends RetryLimited {
public ExponentialBackoffRetry(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
index 81f3a9b..35a45b4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.io.retry.RetryPolicies.RETRY_FOREVER;
import static org.apache.hadoop.io.retry.RetryPolicies.TRY_ONCE_THEN_FAIL;
import static org.apache.hadoop.io.retry.RetryPolicies.retryByException;
import static org.apache.hadoop.io.retry.RetryPolicies.retryByRemoteException;
+import static org.apache.hadoop.io.retry.RetryPolicies.retryOtherThanRemoteException;
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithFixedSleep;
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumCountWithProportionalSleep;
import static org.apache.hadoop.io.retry.RetryPolicies.retryUpToMaximumTimeWithFixedSleep;
@@ -29,6 +30,7 @@ import static org.apache.hadoop.io.retry.RetryPolicies.retryForeverWithFixedSlee
import static org.apache.hadoop.io.retry.RetryPolicies.exponentialBackoffRetry;
import static org.junit.Assert.*;
+import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Callable;
@@ -213,8 +215,29 @@ public class TestRetryProxy {
} catch (RemoteException e) {
// expected
}
- }
-
+ }
+
+ @Test
+ public void testRetryOtherThanRemoteException() throws Throwable {
+ Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
+ Collections.<Class<? extends Exception>, RetryPolicy>singletonMap(
+ IOException.class, RETRY_FOREVER);
+
+ UnreliableInterface unreliable = (UnreliableInterface)
+ RetryProxy.create(UnreliableInterface.class, unreliableImpl,
+ retryOtherThanRemoteException(TRY_ONCE_THEN_FAIL,
+ exceptionToPolicyMap));
+ // should retry with local IOException.
+ unreliable.failsOnceWithIOException();
+ try {
+ // won't get retry on remote exception
+ unreliable.failsOnceWithRemoteException();
+ fail("Should fail");
+ } catch (RemoteException e) {
+ // expected
+ }
+ }
+
@Test
public void testRetryInterruptible() throws Throwable {
final UnreliableInterface unreliable = (UnreliableInterface)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
index ce9c16e..9387772 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
@@ -26,6 +26,8 @@ class UnreliableImplementation implements UnreliableInterface {
private int failsOnceInvocationCount,
failsOnceWithValueInvocationCount,
+ failsOnceIOExceptionInvocationCount,
+ failsOnceRemoteExceptionInvocationCount,
failsTenTimesInvocationCount,
succeedsOnceThenFailsCount,
succeedsOnceThenFailsIdempotentCount,
@@ -90,6 +92,21 @@ class UnreliableImplementation implements UnreliableInterface {
}
@Override
+ public void failsOnceWithIOException() throws IOException {
+ if (failsOnceIOExceptionInvocationCount++ == 0) {
+ throw new IOException("test exception for failsOnceWithIOException");
+ }
+ }
+
+ @Override
+ public void failsOnceWithRemoteException() throws RemoteException {
+ if (failsOnceRemoteExceptionInvocationCount++ == 0) {
+ throw new RemoteException(IOException.class.getName(),
+ "test exception for failsOnceWithRemoteException");
+ }
+ }
+
+ @Override
public void failsTenTimesThenSucceeds() throws UnreliableException {
if (failsTenTimesInvocationCount++ < 10) {
throw new UnreliableException();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
index 3fbe11a..6c9c153 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
@@ -54,6 +54,9 @@ public interface UnreliableInterface {
void alwaysFailsWithFatalException() throws FatalException;
void alwaysFailsWithRemoteFatalException() throws RemoteException;
+ void failsOnceWithIOException() throws IOException;
+ void failsOnceWithRemoteException() throws RemoteException;
+
void failsOnceThenSucceeds() throws UnreliableException;
boolean failsOnceThenSucceedsWithReturnValue() throws UnreliableException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 57cdb15..8dc1fd0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1034,6 +1034,9 @@ Release 2.8.0 - UNRELEASED
YARN-4130. Duplicate declaration of ApplicationId in RMAppManager#submitApplication method.
(Kai Sasaki via rohithsharmaks)
+ YARN-4288. Fixed RMProxy to retry on IOException from local host.
+ (Junping Du via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4169996/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index be08f2f..23e1691 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -250,8 +250,10 @@ public class RMProxy<T> {
exceptionToPolicyMap.put(ConnectTimeoutException.class, retryPolicy);
exceptionToPolicyMap.put(RetriableException.class, retryPolicy);
exceptionToPolicyMap.put(SocketException.class, retryPolicy);
-
- return RetryPolicies.retryByException(
+ // YARN-4288: local IOException is also possible.
+ exceptionToPolicyMap.put(IOException.class, retryPolicy);
+ // Not retry on remote IO exception.
+ return RetryPolicies.retryOtherThanRemoteException(
RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
}
}