You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ac...@apache.org on 2012/08/31 00:46:46 UTC
svn commit: r1379203 - in /hadoop/common/branches/branch-1: CHANGES.txt
src/core/org/apache/hadoop/io/retry/RetryUtils.java
Author: acmurthy
Date: Thu Aug 30 22:46:46 2012
New Revision: 1379203
URL: http://svn.apache.org/viewvc?rev=1379203&view=rev
Log:
HADOOP-8748. Refactor DFSClient retry utility methods to a new class in org.apache.hadoop.io.retry. Contributed by Arun C Murthy.
Added:
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1379203&r1=1379202&r2=1379203&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Thu Aug 30 22:46:46 2012
@@ -106,6 +106,9 @@ Release 1.2.0 - unreleased
MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x
(Koji Noguchi via tgraves)
+ HADOOP-8748. Refactor DFSClient retry utility methods to a new class in
+ org.apache.hadoop.io.retry. Contributed by Arun C Murthy.
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Added: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java?rev=1379203&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java (added)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/io/retry/RetryUtils.java Thu Aug 30 22:46:46 2012
@@ -0,0 +1,149 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RemoteException;
+
+public class RetryUtils {
+ public static final Log LOG = LogFactory.getLog(RetryUtils.class);
+
+ /**
+ * Return the default retry policy set in conf.
+ *
+ * If the value retryPolicyEnabledKey is set to false in conf,
+ * use TRY_ONCE_THEN_FAIL.
+ *
+ * Otherwise, get the MultipleLinearRandomRetry policy specified in the conf
+ * and then
+ * (1) use multipleLinearRandomRetry for
+ * - remoteExceptionToRetry, or
+ * - IOException other than RemoteException; and
+ * (2) use TRY_ONCE_THEN_FAIL for
+ * - RemoteException other than remoteExceptionToRetry, or
+ * - non-IOException.
+ *
+ * @param conf
+ * @param retryPolicyEnabledKey conf property key for enabling retry
+ * @param defaultRetryPolicyEnabled default retryPolicyEnabledKey conf value
+ * @param retryPolicySpecKey conf property key for retry policy spec
+ * @param defaultRetryPolicySpec default retryPolicySpecKey conf value
+ * @param remoteExceptionToRetry The particular RemoteException to retry
+ * @return the default retry policy.
+ */
+ public static RetryPolicy getDefaultRetryPolicy(
+ Configuration conf,
+ String retryPolicyEnabledKey,
+ boolean defaultRetryPolicyEnabled,
+ String retryPolicySpecKey,
+ String defaultRetryPolicySpec,
+ final Class<? extends Exception> remoteExceptionToRetry
+ ) {
+
+ final RetryPolicy multipleLinearRandomRetry =
+ getMultipleLinearRandomRetry(
+ conf,
+ retryPolicyEnabledKey, defaultRetryPolicyEnabled,
+ retryPolicySpecKey, defaultRetryPolicySpec
+ );
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("multipleLinearRandomRetry = " + multipleLinearRandomRetry);
+ }
+
+ if (multipleLinearRandomRetry == null) {
+ //no retry
+ return RetryPolicies.TRY_ONCE_THEN_FAIL;
+ } else {
+ return new RetryPolicy() {
+ @Override
+ public boolean shouldRetry(Exception e, int retries) throws Exception {
+ //see (1) and (2) in the javadoc of this method.
+ final RetryPolicy p;
+ if (e instanceof RemoteException) {
+ final RemoteException re = (RemoteException)e;
+ p = remoteExceptionToRetry.getName().equals(re.getClassName())?
+ multipleLinearRandomRetry: RetryPolicies.TRY_ONCE_THEN_FAIL;
+ } else if (e instanceof IOException) {
+ p = multipleLinearRandomRetry;
+ } else { //non-IOException
+ p = RetryPolicies.TRY_ONCE_THEN_FAIL;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RETRY " + retries + ") policy="
+ + p.getClass().getSimpleName() + ", exception=" + e);
+ }
+ return p.shouldRetry(e, retries);
+ }
+
+ @Override
+ public String toString() {
+ return "RetryPolicy[" + multipleLinearRandomRetry + ", "
+ + RetryPolicies.TRY_ONCE_THEN_FAIL.getClass().getSimpleName()
+ + "]";
+ }
+ };
+ }
+ }
+
+ /**
+ * Return the MultipleLinearRandomRetry policy specified in the conf,
+ * or null if the feature is disabled.
+ * If the policy is specified in the conf but the policy cannot be parsed,
+ * the default policy is returned.
+ *
+ * Retry policy spec:
+ * N pairs of sleep-time and number-of-retries "s1,n1,s2,n2,..."
+ *
+ * @param conf
+ * @param retryPolicyEnabledKey conf property key for enabling retry
+ * @param defaultRetryPolicyEnabled default retryPolicyEnabledKey conf value
+ * @param retryPolicySpecKey conf property key for retry policy spec
+ * @param defaultRetryPolicySpec default retryPolicySpecKey conf value
+ * @return the MultipleLinearRandomRetry policy specified in the conf,
+ * or null if the feature is disabled.
+ */
+ public static RetryPolicy getMultipleLinearRandomRetry(
+ Configuration conf,
+ String retryPolicyEnabledKey,
+ boolean defaultRetryPolicyEnabled,
+ String retryPolicySpecKey,
+ String defaultRetryPolicySpec
+ ) {
+ final boolean enabled =
+ conf.getBoolean(retryPolicyEnabledKey, defaultRetryPolicyEnabled);
+ if (!enabled) {
+ return null;
+ }
+
+ final String policy = conf.get(retryPolicySpecKey, defaultRetryPolicySpec);
+
+ final RetryPolicy r =
+ RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString(
+ policy);
+ return (r != null) ?
+ r :
+ RetryPolicies.MultipleLinearRandomRetry.parseCommaSeparatedString(
+ defaultRetryPolicySpec);
+ }
+}