You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ka...@apache.org on 2014/05/30 17:24:49 UTC
svn commit: r1598630 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/resources/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcem...
Author: kasha
Date: Fri May 30 15:24:49 2014
New Revision: 1598630
URL: http://svn.apache.org/r1598630
Log:
YARN-2054. Better defaults for YARN ZK configs for retries and retry-inteval when HA is enabled. (kasha)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1598630&r1=1598629&r2=1598630&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri May 30 15:24:49 2014
@@ -117,6 +117,9 @@ Release 2.5.0 - UNRELEASED
YARN-596. Use scheduling policies throughout the queue hierarchy to decide
which containers to preempt (Wei Yan via Sandy Ryza)
+ YARN-2054. Better defaults for YARN ZK configs for retries and retry-inteval
+ when HA is enabled. (kasha)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1598630&r1=1598629&r2=1598630&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri May 30 15:24:49 2014
@@ -324,11 +324,11 @@ public class YarnConfiguration extends C
public static final String RM_ZK_ADDRESS = RM_ZK_PREFIX + "address";
public static final String RM_ZK_NUM_RETRIES = RM_ZK_PREFIX + "num-retries";
- public static final int DEFAULT_ZK_RM_NUM_RETRIES = 500;
+ public static final int DEFAULT_ZK_RM_NUM_RETRIES = 1000;
public static final String RM_ZK_RETRY_INTERVAL_MS =
RM_ZK_PREFIX + "retry-interval-ms";
- public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 2000;
+ public static final long DEFAULT_RM_ZK_RETRY_INTERVAL_MS = 1000;
public static final String RM_ZK_TIMEOUT_MS = RM_ZK_PREFIX + "timeout-ms";
public static final int DEFAULT_RM_ZK_TIMEOUT_MS = 10000;
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1598630&r1=1598629&r2=1598630&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri May 30 15:24:49 2014
@@ -309,14 +309,17 @@
<property>
<description>Number of times RM tries to connect to ZooKeeper.</description>
<name>yarn.resourcemanager.zk-num-retries</name>
- <value>500</value>
+ <value>1000</value>
</property>
<property>
<description>Retry interval in milliseconds when connecting to ZooKeeper.
+ When HA is enabled, the value here is NOT used. It is generated
+ automatically from yarn.resourcemanager.zk-timeout-ms and
+ yarn.resourcemanager.zk-num-retries.
</description>
<name>yarn.resourcemanager.zk-retry-interval-ms</name>
- <value>2000</value>
+ <value>1000</value>
</property>
<property>
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1598630&r1=1598629&r2=1598630&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Fri May 30 15:24:49 2014
@@ -90,7 +90,9 @@ public class ZKRMStateStore extends RMSt
private String zkHostPort = null;
private int zkSessionTimeout;
- private long zkRetryInterval;
+
+ @VisibleForTesting
+ long zkRetryInterval;
private List<ACL> zkAcl;
private List<ZKUtil.ZKAuthInfo> zkAuths;
@@ -199,9 +201,14 @@ public class ZKRMStateStore extends RMSt
zkSessionTimeout =
conf.getInt(YarnConfiguration.RM_ZK_TIMEOUT_MS,
YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
- zkRetryInterval =
- conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS,
- YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS);
+
+ if (HAUtil.isHAEnabled(conf)) {
+ zkRetryInterval = zkSessionTimeout / numRetries;
+ } else {
+ zkRetryInterval =
+ conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS);
+ }
zkAcl = RMZKUtils.getZKAcls(conf);
zkAuths = RMZKUtils.getZKAuths(conf);
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java?rev=1598630&r1=1598629&r2=1598630&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Fri May 30 15:24:49 2014
@@ -41,6 +41,7 @@ import java.security.NoSuchAlgorithmExce
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -203,7 +204,7 @@ public class TestZKRMStateStoreZKClientC
LOG.error(error, e);
fail(error);
}
- Assert.assertEquals("newBytes", new String(ret));
+ assertEquals("newBytes", new String(ret));
}
@Test(timeout = 20000)
@@ -232,7 +233,7 @@ public class TestZKRMStateStoreZKClientC
try {
byte[] ret = store.getDataWithRetries(path, false);
- Assert.assertEquals("bytes", new String(ret));
+ assertEquals("bytes", new String(ret));
} catch (Exception e) {
String error = "New session creation failed";
LOG.error(error, e);
@@ -281,4 +282,24 @@ public class TestZKRMStateStoreZKClientC
zkClientTester.getRMStateStore(conf);
}
+
+ @Test
+ public void testZKRetryInterval() throws Exception {
+ TestZKClient zkClientTester = new TestZKClient();
+ YarnConfiguration conf = new YarnConfiguration();
+
+ ZKRMStateStore store =
+ (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
+ assertEquals(YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS,
+ store.zkRetryInterval);
+ store.stop();
+
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ store =
+ (ZKRMStateStore) zkClientTester.getRMStateStore(conf);
+ assertEquals(YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS /
+ YarnConfiguration.DEFAULT_ZK_RM_NUM_RETRIES,
+ store.zkRetryInterval);
+ store.stop();
+ }
}