You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by um...@apache.org on 2020/04/25 00:19:48 UTC
[hadoop-ozone] branch master updated: HDDS-3465. OM Failover retry
happens too quickly when new leader suggested and retrying on same OM.
(#859)
This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4b1fa10 HDDS-3465. OM Failover retry happens too quickly when new leader suggested and retrying on same OM. (#859)
4b1fa10 is described below
commit 4b1fa1099b3996525c86bdf1ba14d9a926667166
Author: Uma Maheswara Rao G <um...@apache.org>
AuthorDate: Fri Apr 24 17:19:41 2020 -0700
HDDS-3465. OM Failover retry happens too quickly when new leader suggested and retrying on same OM. (#859)
---
.../ozone/om/ha/OMFailoverProxyProvider.java | 3 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 6 +
.../ozone/om/ha/TestOMFailoverProxyProvider.java | 160 +++++++++++++++++++++
.../hadoop/ozone/MiniOzoneHAClusterImpl.java | 1 +
.../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 33 ++++-
5 files changed, 202 insertions(+), 1 deletion(-)
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
index f2ddc7d..ec69175 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/ha/OMFailoverProxyProvider.java
@@ -317,10 +317,13 @@ public class OMFailoverProxyProvider implements
synchronized boolean updateLeaderOMNodeId(String newLeaderOMNodeId) {
if (!currentProxyOMNodeId.equals(newLeaderOMNodeId)) {
if (omProxies.containsKey(newLeaderOMNodeId)) {
+ lastAttemptedOM = currentProxyOMNodeId;
currentProxyOMNodeId = newLeaderOMNodeId;
currentProxyIndex = omNodeIDList.indexOf(currentProxyOMNodeId);
return true;
}
+ } else {
+ lastAttemptedOM = currentProxyOMNodeId;
}
return false;
}
diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index ec355e9..db18688 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -263,6 +263,12 @@ public final class OzoneManagerProtocolClientSideTranslatorPB
if (leaderNotReadyException != null) {
FAILOVER_PROXY_PROVIDER_LOG.info("RetryProxy: {}",
leaderNotReadyException.getMessage());
+ // HDDS-3465. OM index will not change, but LastOmID will be
+ // updated to currentOMId, so that wiatTime calculation will
+ // know lastOmID and currentID are same and need to increment
+ // wait time in between.
+ omFailoverProxyProvider.performFailoverIfRequired(
+ omFailoverProxyProvider.getCurrentProxyOMNodeId());
return getRetryAction(RetryDecision.FAILOVER_AND_RETRY, failovers);
}
}
diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
new file mode 100644
index 0000000..20e51d3
--- /dev/null
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/ha/TestOMFailoverProxyProvider.java
@@ -0,0 +1,160 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om.ha;
+
+import java.util.Collection;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.Before;
+
+import java.util.StringJoiner;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.ozone.OmUtils;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_NODES_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT;
+
+/**
+ * Tests OMFailoverProxyProvider failover behaviour.
+ */
+public class TestOMFailoverProxyProvider {
+ private final static String OM_SERVICE_ID = "om-service-test1";
+ private final static String NODE_ID_BASE_STR = "omNode-";
+ private final static String DUMMY_NODE_ADDR = "0.0.0.0:8080";
+ private OMFailoverProxyProvider provider;
+ private long waitBetweenRetries;
+ private int numNodes = 3;
+ private OzoneConfiguration config;
+
+ @Before
+ public void init() throws Exception {
+ config = new OzoneConfiguration();
+ waitBetweenRetries = config.getLong(
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY,
+ OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
+ StringJoiner allNodeIds = new StringJoiner(",");
+ for (int i = 1; i <= numNodes; i++) {
+ String nodeId = NODE_ID_BASE_STR + i;
+ config.set(OmUtils.addKeySuffixes(OZONE_OM_ADDRESS_KEY, OM_SERVICE_ID,
+ nodeId), DUMMY_NODE_ADDR);
+ allNodeIds.add(nodeId);
+ }
+ config.set(OmUtils.addKeySuffixes(OZONE_OM_NODES_KEY, OM_SERVICE_ID),
+ allNodeIds.toString());
+ provider = new OMFailoverProxyProvider(config,
+ UserGroupInformation.getCurrentUser(), OM_SERVICE_ID);
+ }
+
+ /**
+ * Tests waitTime when fail over to next node.
+ */
+ @Test
+ public void testWaitTimeWithNextNode() {
+ failoverToNextNode(numNodes - 1, 0);
+ // After 3 attempts done, wait time should be waitBetweenRetries.
+ failoverToNextNode(1, waitBetweenRetries);
+ // From 4th Attempt waitTime should reset to 0.
+ failoverToNextNode(numNodes - 1, 0);
+ // After 2nd round of 3attempts done, wait time should be
+ // waitBetweenRetries.
+ failoverToNextNode(1, waitBetweenRetries);
+ }
+
+ /**
+ * Tests failover to next node and same node.
+ */
+ @Test
+ public void testWaitTimeWithNextNodeAndSameNodeFailover() {
+ failoverToNextNode(1, 0);
+ // 1 Failover attempt to same OM, waitTime should increase.
+ failoverToSameNode(2);
+ }
+
+ /**
+ * Tests wait time should reset in the following case:
+ * 1. Do a couple same node failover attempts.
+ * 2. Next node failover should reset wait time to 0.
+ */
+ @Test
+ public void testWaitTimeResetWhenNextNodeFailoverAfterSameNode() {
+ // 2 failover attempts to same OM, waitTime should increase.
+ failoverToSameNode(2);
+ // Failover to next node, should reset waitTime to 0.
+ failoverToNextNode(1, 0);
+ }
+
+ /**
+ * Tests wait time should be 0 in the following case:
+ * 1. Do failover to suggest new node.
+ * 2. WaitTime should be 0.
+ */
+ @Test
+ public void testWaitTimeWithSuggestedNewNode() {
+ Collection<String> allNodeIds = config.getTrimmedStringCollection(OmUtils.
+ addKeySuffixes(OZONE_OM_NODES_KEY, OM_SERVICE_ID));
+ allNodeIds.remove(provider.getCurrentProxyOMNodeId());
+ Assert.assertTrue("This test needs at least 2 OMs",
+ allNodeIds.size() > 0);
+ provider.performFailoverIfRequired(allNodeIds.iterator().next());
+ Assert.assertEquals(0, provider.getWaitTime());
+ }
+
+ /**
+ * Tests waitTime reset after same node failover.
+ */
+ @Test
+ public void testWaitTimeResetWhenAllNodeFailoverAndSameNode() {
+ // Next node failover wait time should be 0.
+ failoverToNextNode(numNodes - 1, 0);
+ // Once all numNodes failover done, waitTime should be waitBetweenRetries
+ failoverToNextNode(1, waitBetweenRetries);
+ // 4 failover attempts to same OM, waitTime should increase.
+ failoverToSameNode(4);
+ // Next node failover should reset wait time.
+ failoverToNextNode(numNodes - 1, 0);
+ failoverToNextNode(1, waitBetweenRetries);
+ }
+
+ /**
+ * Failover to next node and wait time should be same as waitTimeAfter.
+ */
+ private void failoverToNextNode(int numNextNodeFailoverTimes,
+ long waitTimeAfter) {
+ for (int attempt = 0; attempt < numNextNodeFailoverTimes; attempt++) {
+ provider.performFailoverToNextProxy();
+ Assert.assertEquals(waitTimeAfter, provider.getWaitTime());
+ }
+ }
+
+ /**
+ * Failover to same node and wait time will be attempt*waitBetweenRetries.
+ */
+ private void failoverToSameNode(int numSameNodeFailoverTimes) {
+ for (int attempt = 1; attempt <= numSameNodeFailoverTimes; attempt++) {
+ provider.performFailoverIfRequired(provider.getCurrentProxyOMNodeId());
+ Assert.assertEquals(attempt * waitBetweenRetries,
+ provider.getWaitTime());
+ }
+ }
+}
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
index ac565f6..4548717 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneHAClusterImpl.java
@@ -249,6 +249,7 @@ public class MiniOzoneHAClusterImpl extends MiniOzoneClusterImpl {
public void stopOzoneManager(String omNodeId) {
ozoneManagerMap.get(omNodeId).stop();
+ ozoneManagerMap.get(omNodeId).join();
}
/**
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
index fe25841..2feccef 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHA.java
@@ -471,7 +471,6 @@ public class TestOzoneManagerHA {
Assert.assertEquals(data, new String(fileContent));
}
- @Ignore("This test failing randomly and triggering HDDS-3465.")
@Test
public void testMultipartUploadWithOneOmNodeDown() throws Exception {
@@ -503,6 +502,38 @@ public class TestOzoneManagerHA {
Assert.assertTrue(leaderOMNodeId != newLeaderOMNodeId);
}
+ /**
+ * 1. Stop one of the OM
+ * 2. make a call to OM, this will make failover attempts to find new node.
+ * a) if LE finishes but leader not ready, it retries to same node
+ * b) if LE not done, it will failover to new node and check
+ * 3. Try failover to same OM explicitly.
+ * Now #3 should wait additional waitBetweenRetries time.
+ * LE: Leader Election.
+ */
+ @Test
+ public void testIncrementalWaitTimeWithSameNodeFailover() throws Exception {
+ long waitBetweenRetries = conf.getLong(
+ OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_KEY,
+ OzoneConfigKeys.OZONE_CLIENT_WAIT_BETWEEN_RETRIES_MILLIS_DEFAULT);
+ OMFailoverProxyProvider omFailoverProxyProvider =
+ objectStore.getClientProxy().getOMProxyProvider();
+
+ // The OMFailoverProxyProvider will point to the current leader OM node.
+ String leaderOMNodeId = omFailoverProxyProvider.getCurrentProxyOMNodeId();
+
+ cluster.stopOzoneManager(leaderOMNodeId);
+ Thread.sleep(NODE_FAILURE_TIMEOUT * 2);
+ createKeyTest(true); // failover should happen to new node
+
+ long numTimesTriedToSameNode = omFailoverProxyProvider.getWaitTime()
+ / waitBetweenRetries;
+ omFailoverProxyProvider.performFailoverIfRequired(omFailoverProxyProvider.
+ getCurrentProxyOMNodeId());
+ Assert.assertEquals((numTimesTriedToSameNode + 1) * waitBetweenRetries,
+ omFailoverProxyProvider.getWaitTime());
+ }
+
private String initiateMultipartUpload(OzoneBucket ozoneBucket,
String keyName) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: ozone-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: ozone-commits-help@hadoop.apache.org