You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by lj...@apache.org on 2020/08/04 09:25:39 UTC
[hadoop-ozone] branch master updated: HDDS-3994. Make retry policy
can be set by configuration. (#1231)
This is an automated email from the ASF dual-hosted git repository.
ljain pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ff621c6 HDDS-3994. Make retry policy can be set by configuration. (#1231)
ff621c6 is described below
commit ff621c6f0083f67862a5fbb554f13761ea97113b
Author: maobaolong <30...@qq.com>
AuthorDate: Tue Aug 4 14:54:35 2020 +0530
HDDS-3994. Make retry policy can be set by configuration. (#1231)
---
.../org/apache/hadoop/hdds/ratis/RatisHelper.java | 112 ++++---------------
.../hadoop/hdds/ratis/conf/RatisClientConfig.java | 35 ++++++
.../RequestTypeDependentRetryPolicyCreator.java | 120 +++++++++++++++++++++
.../retrypolicy/RetryLimitedPolicyCreator.java | 47 ++++++++
.../hdds/ratis/retrypolicy/RetryPolicyCreator.java | 29 +++++
.../hdds/ratis/retrypolicy/package-info.java | 23 ++++
6 files changed, 276 insertions(+), 90 deletions(-)
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
index 51113b2..8325f09 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/RatisHelper.java
@@ -20,19 +20,18 @@ package org.apache.hadoop.hdds.ratis;
import java.io.IOException;
import java.security.cert.X509Certificate;
-import java.time.Duration;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
-import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.apache.hadoop.hdds.StringUtils;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
+import org.apache.hadoop.hdds.ratis.retrypolicy.RetryPolicyCreator;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
@@ -40,7 +39,6 @@ import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.ratis.RaftConfigKeys;
import org.apache.ratis.client.RaftClient;
import org.apache.ratis.client.RaftClientConfigKeys;
-import org.apache.ratis.client.retry.RequestTypeDependentRetryPolicy;
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.grpc.GrpcConfigKeys;
import org.apache.ratis.grpc.GrpcFactory;
@@ -50,20 +48,10 @@ import org.apache.ratis.protocol.RaftGroup;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.protocol.RaftPeer;
import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.protocol.GroupMismatchException;
-import org.apache.ratis.protocol.StateMachineException;
-import org.apache.ratis.protocol.NotReplicatedException;
-import org.apache.ratis.protocol.TimeoutIOException;
-import org.apache.ratis.protocol.exceptions.ResourceUnavailableException;
-import org.apache.ratis.retry.ExponentialBackoffRetry;
-import org.apache.ratis.retry.MultipleLinearRandomRetry;
-import org.apache.ratis.retry.ExceptionDependentRetry;
-import org.apache.ratis.retry.RetryPolicies;
import org.apache.ratis.retry.RetryPolicy;
import org.apache.ratis.rpc.RpcType;
import org.apache.ratis.rpc.SupportedRpcType;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
-import org.apache.ratis.util.TimeDuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,10 +65,6 @@ public final class RatisHelper {
// Prefix for Ratis Server GRPC and Ratis client conf.
public static final String HDDS_DATANODE_RATIS_PREFIX_KEY = "hdds.ratis";
- private static final Class[] NO_RETRY_EXCEPTIONS =
- new Class[] {NotReplicatedException.class, GroupMismatchException.class,
- StateMachineException.class};
-
/* TODO: use a dummy id for all groups for the moment.
* It should be changed to a unique id for each group.
*/
@@ -288,74 +272,17 @@ public final class RatisHelper {
return tlsConfig;
}
- /**
- * Table mapping exception type to retry policy used for the exception in
- * write and watch request.
- * ---------------------------------------------------------------------------
- * | Exception | RetryPolicy for | RetryPolicy for |
- * | | Write request | Watch request |
- * |-------------------------------------------------------------------------|
- * | NotReplicatedException | NO_RETRY | NO_RETRY |
- * |-------------------------------------------------------------------------|
- * | GroupMismatchException | NO_RETRY | NO_RETRY |
- * |-------------------------------------------------------------------------|
- * | StateMachineException | NO_RETRY | NO_RETRY |
- * |-------------------------------------------------------------------------|
- * | TimeoutIOException | EXPONENTIAL_BACKOFF | NO_RETRY |
- * |-------------------------------------------------------------------------|
- * | ResourceUnavailableException| EXPONENTIAL_BACKOFF | EXPONENTIAL_BACKOFF |
- * |-------------------------------------------------------------------------|
- * | Others | MULTILINEAR_RANDOM | MULTILINEAR_RANDOM |
- * | | _RETRY | _RETRY |
- * ---------------------------------------------------------------------------
- */
public static RetryPolicy createRetryPolicy(ConfigurationSource conf) {
- RatisClientConfig ratisClientConfig = conf
- .getObject(RatisClientConfig.class);
- ExponentialBackoffRetry exponentialBackoffRetry =
- createExponentialBackoffPolicy(ratisClientConfig);
- MultipleLinearRandomRetry multipleLinearRandomRetry =
- MultipleLinearRandomRetry
- .parseCommaSeparated(ratisClientConfig.getMultilinearPolicy());
-
- return RequestTypeDependentRetryPolicy.newBuilder()
- .setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase.WRITE,
- createExceptionDependentPolicy(exponentialBackoffRetry,
- multipleLinearRandomRetry, exponentialBackoffRetry))
- .setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase.WATCH,
- createExceptionDependentPolicy(exponentialBackoffRetry,
- multipleLinearRandomRetry, RetryPolicies.noRetry()))
- .setTimeout(RaftProtos.RaftClientRequestProto.TypeCase.WRITE,
- toTimeDuration(ratisClientConfig.getWriteRequestTimeout()))
- .setTimeout(RaftProtos.RaftClientRequestProto.TypeCase.WATCH,
- toTimeDuration(ratisClientConfig.getWatchRequestTimeout()))
- .build();
- }
-
- private static ExponentialBackoffRetry createExponentialBackoffPolicy(
- RatisClientConfig ratisClientConfig) {
- return ExponentialBackoffRetry.newBuilder()
- .setBaseSleepTime(
- toTimeDuration(ratisClientConfig.getExponentialPolicyBaseSleep()))
- .setMaxSleepTime(
- toTimeDuration(ratisClientConfig.getExponentialPolicyMaxSleep()))
- .build();
- }
-
- private static ExceptionDependentRetry createExceptionDependentPolicy(
- ExponentialBackoffRetry exponentialBackoffRetry,
- MultipleLinearRandomRetry multipleLinearRandomRetry,
- RetryPolicy timeoutPolicy) {
- ExceptionDependentRetry.Builder builder =
- ExceptionDependentRetry.newBuilder();
- for (Class c : NO_RETRY_EXCEPTIONS) {
- builder.setExceptionToPolicy(c, RetryPolicies.noRetry());
+ try {
+ RatisClientConfig scmClientConfig =
+ conf.getObject(RatisClientConfig.class);
+ Class<? extends RetryPolicyCreator> policyClass = getClass(
+ scmClientConfig.getRetryPolicy(),
+ RetryPolicyCreator.class);
+ return policyClass.newInstance().create(conf);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
- return builder.setExceptionToPolicy(ResourceUnavailableException.class,
- exponentialBackoffRetry)
- .setExceptionToPolicy(TimeoutIOException.class, timeoutPolicy)
- .setDefaultPolicy(multipleLinearRandomRetry)
- .build();
}
public static Long getMinReplicatedIndex(
@@ -364,12 +291,17 @@ public final class RatisHelper {
.min(Long::compareTo).orElse(null);
}
- private static TimeDuration toTimeDuration(Duration duration) {
- return toTimeDuration(duration.toMillis());
- }
-
- private static TimeDuration toTimeDuration(long milliseconds) {
- return TimeDuration.valueOf(milliseconds, TimeUnit.MILLISECONDS);
+ private static <U> Class<? extends U> getClass(String name,
+ Class<U> xface) {
+ try {
+ Class<?> theClass = Class.forName(name);
+ if (!xface.isAssignableFrom(theClass)) {
+ throw new RuntimeException(theClass + " not " + xface.getName());
+ } else {
+ return theClass.asSubclass(xface);
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
}
-
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java
index dc07fdd..7db6059 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/conf/RatisClientConfig.java
@@ -185,4 +185,39 @@ public class RatisClientConfig {
public void setExponentialPolicyMaxSleep(Duration duration) {
exponentialPolicyMaxSleepInMs = duration.toMillis();
}
+
+ @Config(key = "client.retrylimited.retry.interval",
+ defaultValue = "1s",
+ type = ConfigType.TIME,
+ tags = { OZONE, CLIENT, PERFORMANCE },
+ description = "Interval between successive retries for "
+ + "a ratis client request.")
+ private long retrylimitedRetryInterval;
+
+ public long getRetrylimitedRetryInterval() {
+ return retrylimitedRetryInterval;
+ }
+
+ @Config(key = "client.retrylimited.max.retries",
+ defaultValue = "180",
+ type = ConfigType.INT,
+ tags = { OZONE, CLIENT, PERFORMANCE },
+ description = "Number of retries for ratis client request.")
+ private int retrylimitedMaxRetries;
+
+ public int getRetrylimitedMaxRetries() {
+ return retrylimitedMaxRetries;
+ }
+
+ @Config(key = "client.retry.policy",
+ defaultValue = "org.apache.hadoop.hdds.ratis.retrypolicy."
+ + "RequestTypeDependentRetryPolicyCreator",
+ type = ConfigType.STRING,
+ tags = { OZONE, CLIENT, PERFORMANCE },
+ description = "The class name of the policy for retry.")
+ private String retryPolicy;
+
+ public String getRetryPolicy() {
+ return retryPolicy;
+ }
}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RequestTypeDependentRetryPolicyCreator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RequestTypeDependentRetryPolicyCreator.java
new file mode 100644
index 0000000..fe92f32
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RequestTypeDependentRetryPolicyCreator.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.ratis.retrypolicy;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
+import org.apache.ratis.client.retry.RequestTypeDependentRetryPolicy;
+import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.protocol.GroupMismatchException;
+import org.apache.ratis.protocol.NotReplicatedException;
+import org.apache.ratis.protocol.StateMachineException;
+import org.apache.ratis.protocol.TimeoutIOException;
+import org.apache.ratis.protocol.exceptions.ResourceUnavailableException;
+import org.apache.ratis.retry.*;
+import org.apache.ratis.util.TimeDuration;
+
+import java.time.Duration;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Table mapping exception type to retry policy used for the exception in
+ * write and watch request.
+ * ---------------------------------------------------------------------------
+ * | Exception | RetryPolicy for | RetryPolicy for |
+ * | | Write request | Watch request |
+ * |-------------------------------------------------------------------------|
+ * | NotReplicatedException | NO_RETRY | NO_RETRY |
+ * |-------------------------------------------------------------------------|
+ * | GroupMismatchException | NO_RETRY | NO_RETRY |
+ * |-------------------------------------------------------------------------|
+ * | StateMachineException | NO_RETRY | NO_RETRY |
+ * |-------------------------------------------------------------------------|
+ * | TimeoutIOException | EXPONENTIAL_BACKOFF | NO_RETRY |
+ * |-------------------------------------------------------------------------|
+ * | ResourceUnavailableException| EXPONENTIAL_BACKOFF | EXPONENTIAL_BACKOFF |
+ * |-------------------------------------------------------------------------|
+ * | Others | MULTILINEAR_RANDOM | MULTILINEAR_RANDOM |
+ * | | _RETRY | _RETRY |
+ * ---------------------------------------------------------------------------
+ */
+public class RequestTypeDependentRetryPolicyCreator
+ implements RetryPolicyCreator {
+
+ private static final Class[] NO_RETRY_EXCEPTIONS =
+ new Class[] {NotReplicatedException.class, GroupMismatchException.class,
+ StateMachineException.class};
+
+ @Override
+ public RetryPolicy create(ConfigurationSource conf) {
+ RatisClientConfig ratisClientConfig = conf
+ .getObject(RatisClientConfig.class);
+ ExponentialBackoffRetry exponentialBackoffRetry =
+ createExponentialBackoffPolicy(ratisClientConfig);
+ MultipleLinearRandomRetry multipleLinearRandomRetry =
+ MultipleLinearRandomRetry
+ .parseCommaSeparated(ratisClientConfig.getMultilinearPolicy());
+
+ return RequestTypeDependentRetryPolicy.newBuilder()
+ .setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase.WRITE,
+ createExceptionDependentPolicy(exponentialBackoffRetry,
+ multipleLinearRandomRetry, exponentialBackoffRetry))
+ .setRetryPolicy(RaftProtos.RaftClientRequestProto.TypeCase.WATCH,
+ createExceptionDependentPolicy(exponentialBackoffRetry,
+ multipleLinearRandomRetry, RetryPolicies.noRetry()))
+ .setTimeout(RaftProtos.RaftClientRequestProto.TypeCase.WRITE,
+ toTimeDuration(ratisClientConfig.getWriteRequestTimeout()))
+ .setTimeout(RaftProtos.RaftClientRequestProto.TypeCase.WATCH,
+ toTimeDuration(ratisClientConfig.getWatchRequestTimeout()))
+ .build();
+ }
+
+ private static ExponentialBackoffRetry createExponentialBackoffPolicy(
+ RatisClientConfig ratisClientConfig) {
+ return ExponentialBackoffRetry.newBuilder()
+ .setBaseSleepTime(
+ toTimeDuration(ratisClientConfig.getExponentialPolicyBaseSleep()))
+ .setMaxSleepTime(
+ toTimeDuration(ratisClientConfig.getExponentialPolicyMaxSleep()))
+ .build();
+ }
+
+ private static ExceptionDependentRetry createExceptionDependentPolicy(
+ ExponentialBackoffRetry exponentialBackoffRetry,
+ MultipleLinearRandomRetry multipleLinearRandomRetry,
+ RetryPolicy timeoutPolicy) {
+ ExceptionDependentRetry.Builder builder =
+ ExceptionDependentRetry.newBuilder();
+ for (Class c : NO_RETRY_EXCEPTIONS) {
+ builder.setExceptionToPolicy(c, RetryPolicies.noRetry());
+ }
+ return builder.setExceptionToPolicy(ResourceUnavailableException.class,
+ exponentialBackoffRetry)
+ .setExceptionToPolicy(TimeoutIOException.class, timeoutPolicy)
+ .setDefaultPolicy(multipleLinearRandomRetry)
+ .build();
+ }
+ private static TimeDuration toTimeDuration(Duration duration) {
+ return toTimeDuration(duration.toMillis());
+ }
+
+ private static TimeDuration toTimeDuration(long milliseconds) {
+ return TimeDuration.valueOf(milliseconds, TimeUnit.MILLISECONDS);
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RetryLimitedPolicyCreator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RetryLimitedPolicyCreator.java
new file mode 100644
index 0000000..5c3b06a
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RetryLimitedPolicyCreator.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.ratis.retrypolicy;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
+import org.apache.ratis.retry.RetryPolicies;
+import org.apache.ratis.retry.RetryPolicy;
+import org.apache.ratis.util.TimeDuration;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * The creator of RetryLimited policy.
+ */
+public class RetryLimitedPolicyCreator implements RetryPolicyCreator {
+
+ @Override
+ public RetryPolicy create(ConfigurationSource conf) {
+ RatisClientConfig scmClientConfig =
+ conf.getObject(RatisClientConfig.class);
+ int maxRetryCount =
+ scmClientConfig.getRetrylimitedMaxRetries();
+ long retryInterval = scmClientConfig.getRetrylimitedRetryInterval();
+ TimeDuration sleepDuration =
+ TimeDuration.valueOf(retryInterval, TimeUnit.MILLISECONDS);
+ RetryPolicy retryPolicy = RetryPolicies
+ .retryUpToMaximumCountWithFixedSleep(maxRetryCount, sleepDuration);
+ return retryPolicy;
+ }
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RetryPolicyCreator.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RetryPolicyCreator.java
new file mode 100644
index 0000000..8057baa
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/RetryPolicyCreator.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.ratis.retrypolicy;
+
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.ratis.retry.RetryPolicy;
+
+/**
+ * The interface of RetryLimited policy creator.
+ */
+public interface RetryPolicyCreator {
+ RetryPolicy create(ConfigurationSource conf);
+}
diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/package-info.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/package-info.java
new file mode 100644
index 0000000..657a2bf
--- /dev/null
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/ratis/retrypolicy/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.ratis.retrypolicy;
+
+/**
+ * This package contains classes related to retry policies.
+ */
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org