You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ac...@apache.org on 2013/08/15 04:37:05 UTC
svn commit: r1514137 - in
/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/
hadoop-yarn/had...
Author: acmurthy
Date: Thu Aug 15 02:37:05 2013
New Revision: 1514137
URL: http://svn.apache.org/r1514137
Log:
Merge -c 1514135 from trunk to branch-2.1-beta to fix YARN-1056. Remove dual use of string 'resourcemanager' in yarn.resourcemanager.connect.{max.wait.secs|retry_interval.secs}. Contributed by Karthik Kambatla.
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt?rev=1514137&r1=1514136&r2=1514137&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/CHANGES.txt Thu Aug 15 02:37:05 2013
@@ -842,6 +842,10 @@ Release 2.1.0-beta - 2013-08-06
YARN-1043. Push all metrics consistently. (Jian He via acmurthy)
+ YARN-1056. Remove dual use of string 'resourcemanager' in
+ yarn.resourcemanager.connect.{max.wait.secs|retry_interval.secs}
+ (Karthik Kambatla via acmurthy)
+
Release 2.0.5-alpha - 06/06/2013
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1514137&r1=1514136&r2=1514137&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Thu Aug 15 02:37:05 2013
@@ -259,7 +259,7 @@ public class YarnConfiguration extends C
/** URI for FileSystemRMStateStore */
public static final String FS_RM_STATE_STORE_URI =
- RM_PREFIX + "fs.rm-state-store.uri";
+ RM_PREFIX + "fs.state-store.uri";
/** The maximum number of completed applications RM keeps. */
public static final String RM_MAX_COMPLETED_APPLICATIONS =
@@ -655,19 +655,17 @@ public class YarnConfiguration extends C
public static final long DEFAULT_NM_PROCESS_KILL_WAIT_MS =
2000;
- /** Max time to wait to establish a connection to RM
- */
- public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS =
- RM_PREFIX + "resourcemanager.connect.max.wait.secs";
- public static final int DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS =
- 15*60;
-
- /** Time interval between each attempt to connect to RM
- */
- public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS =
- RM_PREFIX + "resourcemanager.connect.retry_interval.secs";
- public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS
- = 30;
+ /** Max time to wait to establish a connection to RM */
+ public static final String RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
+ RM_PREFIX + "connect.max-wait.ms";
+ public static final int DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS =
+ 15 * 60 * 1000;
+
+ /** Time interval between each attempt to connect to RM */
+ public static final String RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS =
+ RM_PREFIX + "connect.retry-interval.ms";
+ public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
+ = 30 * 1000;
/**
* CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java?rev=1514137&r1=1514136&r2=1514137&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java Thu Aug 15 02:37:05 2013
@@ -35,14 +35,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryProxy;
-import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import com.google.common.annotations.VisibleForTesting;
@@ -79,38 +75,36 @@ public class RMProxy<T> {
public static RetryPolicy createRetryPolicy(Configuration conf) {
long rmConnectWaitMS =
conf.getInt(
- YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS,
- YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS)
- * 1000;
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+ YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_MAX_WAIT_MS);
long rmConnectionRetryIntervalMS =
conf.getLong(
- YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
YarnConfiguration
- .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS)
- * 1000;
+ .DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS);
if (rmConnectionRetryIntervalMS < 0) {
throw new YarnRuntimeException("Invalid Configuration. " +
- YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS +
+ YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
" should not be negative.");
}
- boolean waitForEver = (rmConnectWaitMS == -1000);
+ boolean waitForEver = (rmConnectWaitMS == -1);
if (waitForEver) {
return RetryPolicies.RETRY_FOREVER;
} else {
if (rmConnectWaitMS < 0) {
throw new YarnRuntimeException("Invalid Configuration. "
- + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS
+ + YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
+ " can be -1, but can not be other negative numbers");
}
// try connect once
if (rmConnectWaitMS < rmConnectionRetryIntervalMS) {
- LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS
+ LOG.warn(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS
+ " is smaller than "
- + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS
+ + YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
+ ". Only try connect once.");
rmConnectWaitMS = 0;
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1514137&r1=1514136&r2=1514137&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Thu Aug 15 02:37:05 2013
@@ -141,6 +141,20 @@
</property>
<property>
+ <description>Maximum time to wait to establish connection to
+ ResourceManager.</description>
+ <name>yarn.resourcemanager.connect.max-wait.ms</name>
+ <value>900000</value>
+ </property>
+
+ <property>
+ <description>How often to try connecting to the
+ ResourceManager.</description>
+ <name>yarn.resourcemanager.connect.retry-interval.ms</name>
+ <value>30000</value>
+ </property>
+
+ <property>
<description>The maximum number of application attempts. It's a global
setting for all application masters. Each application master can specify
its individual maximum number of application attempts via the API, but the
@@ -249,7 +263,7 @@
RM state will be stored. This must be supplied when using
org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
as the value for yarn.resourcemanager.store.class</description>
- <name>yarn.resourcemanager.fs.rm-state-store.uri</name>
+ <name>yarn.resourcemanager.fs.state-store.uri</name>
<value>${hadoop.tmp.dir}/yarn/system/rmstore</value>
<!--value>hdfs://localhost:9000/rmstore</value-->
</property>
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1514137&r1=1514136&r2=1514137&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Thu Aug 15 02:37:05 2013
@@ -957,15 +957,14 @@ public class TestNodeStatusUpdater {
@Test (timeout = 150000)
public void testNMConnectionToRM() throws Exception {
final long delta = 50000;
- final long connectionWaitSecs = 5;
- final long connectionRetryIntervalSecs = 1;
+ final long connectionWaitMs = 5000;
+ final long connectionRetryIntervalMs = 1000;
//Waiting for rmStartIntervalMS, RM will be started
final long rmStartIntervalMS = 2*1000;
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS,
- connectionWaitSecs);
- conf.setLong(YarnConfiguration
- .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
- connectionRetryIntervalSecs);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+ connectionWaitMs);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ connectionRetryIntervalMs);
//Test NM try to connect to RM Several times, but finally fail
NodeManagerWithCustomNodeStatusUpdater nmWithUpdater;
@@ -987,15 +986,15 @@ public class TestNodeStatusUpdater {
} catch(Exception e) {
long t = System.currentTimeMillis();
long duration = t - waitStartTime;
- boolean waitTimeValid = (duration >= connectionWaitSecs * 1000)
- && (duration < (connectionWaitSecs * 1000 + delta));
+ boolean waitTimeValid = (duration >= connectionWaitMs)
+ && (duration < (connectionWaitMs + delta));
if(!waitTimeValid) {
//either the exception was too early, or it had a different cause.
//reject with the inner stack trace
throw new Exception("NM should have tried re-connecting to RM during " +
- "period of at least " + connectionWaitSecs + " seconds, but " +
- "stopped retrying within " + (connectionWaitSecs + delta/1000) +
- " seconds: " + e, e);
+ "period of at least " + connectionWaitMs + " ms, but " +
+ "stopped retrying within " + (connectionWaitMs + delta) +
+ " ms: " + e, e);
}
}
@@ -1149,14 +1148,14 @@ public class TestNodeStatusUpdater {
@Test(timeout = 200000)
public void testNodeStatusUpdaterRetryAndNMShutdown()
throws Exception {
- final long connectionWaitSecs = 1;
- final long connectionRetryIntervalSecs = 1;
+ final long connectionWaitSecs = 1000;
+ final long connectionRetryIntervalMs = 1000;
YarnConfiguration conf = createNMConfig();
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS,
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
connectionWaitSecs);
conf.setLong(YarnConfiguration
- .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
- connectionRetryIntervalSecs);
+ .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ connectionRetryIntervalMs);
conf.setLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 5000);
CyclicBarrier syncBarrier = new CyclicBarrier(2);
nm = new MyNodeManager2(syncBarrier, conf);