You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ji...@apache.org on 2014/02/07 19:42:07 UTC
svn commit: r1565757 - in
/hadoop/common/branches/HDFS-5698/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/
hadoop-yarn/hadoo...
Author: jing9
Date: Fri Feb 7 18:42:06 2014
New Revision: 1565757
URL: http://svn.apache.org/r1565757
Log:
Merging r1565514 through r1565756 from trunk
Modified:
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/CHANGES.txt?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/CHANGES.txt Fri Feb 7 18:42:06 2014
@@ -17,6 +17,9 @@ Trunk - Unreleased
YARN-1499. Fair Scheduler changes for moving apps between queues (Sandy
Ryza)
+ YARN-1497. Command line additions for moving apps between queues (Sandy
+ Ryza)
+
IMPROVEMENTS
OPTIMIZATIONS
@@ -139,6 +142,14 @@ Release 2.4.0 - UNRELEASED
be available across RM failover by making using of a remote
configuration-provider. (Xuan Gong via vinodkv)
+ YARN-1665. Simplify the configuration of RM HA by having better default
+ values. (Xuan Gong via vinodkv)
+
+ YARN-1660. Simplified the RM HA configuration to accept and be able to simply
+ depend just on configuration properties of the form
+ yarn.resourcemanager.hostname.RMID and use the default ports for all service
+ addresses. (Xuan Gong via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/HAUtil.java Fri Feb 7 18:42:06 2014
@@ -101,20 +101,7 @@ public class HAUtil {
for (String id: ids) {
// verify the RM service addresses configurations for every RMIds
for (String prefix : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
- String confKey = null;
- try {
- confKey = addSuffix(prefix, id);
- if (conf.getTrimmed(confKey) == null) {
- throwBadConfigurationException(getNeedToSetValueMessage(confKey));
- }
- } catch (IllegalArgumentException iae) {
- String errmsg = iae.getMessage();
- if (confKey == null) {
- // Error at addSuffix
- errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, id);
- }
- throwBadConfigurationException(errmsg);
- }
+ checkAndSetRMRPCAddress(prefix, id, conf);
}
setValue.append(id);
setValue.append(",");
@@ -249,9 +236,13 @@ public class HAUtil {
@InterfaceAudience.Private
@VisibleForTesting
static String getConfKeyForRMInstance(String prefix, Configuration conf) {
- return YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)
- ? addSuffix(prefix, getRMHAId(conf))
- : prefix;
+ if (!YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS.contains(prefix)) {
+ return prefix;
+ } else {
+ String RMId = getRMHAId(conf);
+ checkAndSetRMRPCAddress(prefix, RMId, conf);
+ return addSuffix(prefix, RMId);
+ }
}
public static String getConfValueForRMInstance(String prefix,
@@ -284,4 +275,30 @@ public class HAUtil {
}
return key + "." + suffix;
}
+
+ private static void checkAndSetRMRPCAddress(String prefix, String RMId,
+ Configuration conf) {
+ String rpcAddressConfKey = null;
+ try {
+ rpcAddressConfKey = addSuffix(prefix, RMId);
+ if (conf.getTrimmed(rpcAddressConfKey) == null) {
+ String hostNameConfKey = addSuffix(YarnConfiguration.RM_HOSTNAME, RMId);
+ String confVal = conf.getTrimmed(hostNameConfKey);
+ if (confVal == null) {
+ throwBadConfigurationException(getNeedToSetValueMessage(
+ hostNameConfKey + " or " + addSuffix(prefix, RMId)));
+ } else {
+ conf.set(addSuffix(prefix, RMId), confVal + ":"
+ + YarnConfiguration.getRMDefaultPortNumber(prefix));
+ }
+ }
+ } catch (IllegalArgumentException iae) {
+ String errmsg = iae.getMessage();
+ if (rpcAddressConfKey == null) {
+ // Error at addSuffix
+ errmsg = getInvalidValueMessage(YarnConfiguration.RM_HA_ID, RMId);
+ }
+ throwBadConfigurationException(errmsg);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Feb 7 18:42:06 2014
@@ -23,6 +23,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
@@ -108,6 +109,8 @@ public class YarnConfiguration extends C
public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id";
+ public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
+
/** The address of the applications manager interface in the RM.*/
public static final String RM_ADDRESS =
RM_PREFIX + "address";
@@ -373,11 +376,11 @@ public class YarnConfiguration extends C
public static final String AUTO_FAILOVER_ENABLED =
AUTO_FAILOVER_PREFIX + "enabled";
- public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = false;
+ public static final boolean DEFAULT_AUTO_FAILOVER_ENABLED = true;
public static final String AUTO_FAILOVER_EMBEDDED =
AUTO_FAILOVER_PREFIX + "embedded";
- public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = false;
+ public static final boolean DEFAULT_AUTO_FAILOVER_EMBEDDED = true;
public static final String AUTO_FAILOVER_ZK_BASE_PATH =
AUTO_FAILOVER_PREFIX + "zk-base-path";
@@ -1139,4 +1142,27 @@ public class YarnConfiguration extends C
}
return super.updateConnectAddr(prefix, addr);
}
+
+ @Private
+ public static int getRMDefaultPortNumber(String addressPrefix) {
+ if (addressPrefix.equals(YarnConfiguration.RM_ADDRESS)) {
+ return YarnConfiguration.DEFAULT_RM_PORT;
+ } else if (addressPrefix.equals(YarnConfiguration.RM_SCHEDULER_ADDRESS)) {
+ return YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT;
+ } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_ADDRESS)) {
+ return YarnConfiguration.DEFAULT_RM_WEBAPP_PORT;
+ } else if (addressPrefix.equals(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS)) {
+ return YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT;
+ } else if (addressPrefix
+ .equals(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS)) {
+ return YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT;
+ } else if (addressPrefix.equals(YarnConfiguration.RM_ADMIN_ADDRESS)) {
+ return YarnConfiguration.DEFAULT_RM_ADMIN_PORT;
+ } else {
+ throw new HadoopIllegalArgumentException(
+ "Invalid RM RPC address Prefix: " + addressPrefix
+ + ". The valid value should be one of "
+ + YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS);
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java Fri Feb 7 18:42:06 2014
@@ -436,4 +436,19 @@ public abstract class YarnClient extends
public abstract List<ContainerReport> getContainers(
ApplicationAttemptId applicationAttemptId) throws YarnException,
IOException;
+
+ /**
+ * <p>
+ * Attempts to move the given application to the given queue.
+ * </p>
+ *
+ * @param appId
+ * Application to move.
+ * @param queue
+ * Queue to place it in to.
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract void moveApplicationAcrossQueues(ApplicationId appId,
+ String queue) throws YarnException, IOException;
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Fri Feb 7 18:42:06 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueUserAclsInfoRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.MoveApplicationAcrossQueuesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
@@ -478,4 +479,12 @@ public class YarnClientImpl extends Yarn
}
throw new YarnException("History service is not enabled.");
}
+
+ @Override
+ public void moveApplicationAcrossQueues(ApplicationId appId,
+ String queue) throws YarnException, IOException {
+ MoveApplicationAcrossQueuesRequest request =
+ MoveApplicationAcrossQueuesRequest.newInstance(appId, queue);
+ rmClient.moveApplicationAcrossQueues(request);
+ }
}
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Fri Feb 7 18:42:06 2014
@@ -61,6 +61,7 @@ public class ApplicationCLI extends Yarn
private static final String APP_TYPE_CMD = "appTypes";
private static final String APP_STATE_CMD = "appStates";
private static final String ALLSTATES_OPTION = "ALL";
+ private static final String QUEUE_CMD = "queue";
public static final String APPLICATION = "application";
public static final String APPLICATION_ATTEMPT = "applicationattempt";
public static final String CONTAINER = "container";
@@ -96,6 +97,10 @@ public class ApplicationCLI extends Yarn
+ "and -appStates to filter applications based on application state");
}
opts.addOption(KILL_CMD, true, "Kills the application.");
+ opts.addOption(MOVE_TO_QUEUE_CMD, true, "Moves the application to a "
+ + "different queue.");
+ opts.addOption(QUEUE_CMD, true, "Works with the movetoqueue command to"
+ + " specify which queue to move an application to.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to "
+ "filter applications based on "
@@ -112,6 +117,8 @@ public class ApplicationCLI extends Yarn
appStateOpt.setArgName("States");
opts.addOption(appStateOpt);
opts.getOption(KILL_CMD).setArgName("Application ID");
+ opts.getOption(MOVE_TO_QUEUE_CMD).setArgName("Application ID");
+ opts.getOption(QUEUE_CMD).setArgName("Queue Name");
opts.getOption(STATUS_CMD).setArgName("Application ID");
int exitCode = -1;
@@ -202,6 +209,13 @@ public class ApplicationCLI extends Yarn
return exitCode;
}
killApplication(cliParser.getOptionValue(KILL_CMD));
+ } else if (cliParser.hasOption(MOVE_TO_QUEUE_CMD)) {
+ if (!cliParser.hasOption(QUEUE_CMD)) {
+ printUsage(opts);
+ return exitCode;
+ }
+ moveApplicationAcrossQueues(cliParser.getOptionValue(MOVE_TO_QUEUE_CMD),
+ cliParser.getOptionValue(QUEUE_CMD));
} else if (cliParser.hasOption(HELP_CMD)) {
printUsage(opts);
return 0;
@@ -366,6 +380,28 @@ public class ApplicationCLI extends Yarn
client.killApplication(appId);
}
}
+
+ /**
+ * Kills the application with the application id as appId
+ *
+ * @param applicationId
+ * @throws YarnException
+ * @throws IOException
+ */
+ private void moveApplicationAcrossQueues(String applicationId, String queue)
+ throws YarnException, IOException {
+ ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
+ ApplicationReport appReport = client.getApplicationReport(appId);
+ if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
+ || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
+ || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
+ sysout.println("Application " + applicationId + " has already finished ");
+ } else {
+ sysout.println("Moving application " + applicationId + " to queue " + queue);
+ client.moveApplicationAcrossQueues(appId, queue);
+ sysout.println("Successfully completed move.");
+ }
+ }
/**
* Prints the application report for an application id.
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/YarnCLI.java Fri Feb 7 18:42:06 2014
@@ -33,6 +33,7 @@ public abstract class YarnCLI extends Co
public static final String STATUS_CMD = "status";
public static final String LIST_CMD = "list";
public static final String KILL_CMD = "kill";
+ public static final String MOVE_TO_QUEUE_CMD = "movetoqueue";
public static final String HELP_CMD = "help";
protected PrintStream sysout;
protected PrintStream syserr;
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java Fri Feb 7 18:42:06 2014
@@ -172,8 +172,6 @@ public class TestRMFailover extends Clie
@Test
public void testAutomaticFailover()
throws YarnException, InterruptedException, IOException {
- conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
- conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_EMBEDDED, true);
conf.set(YarnConfiguration.RM_CLUSTER_ID, "yarn-test-cluster");
conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 2000);
@@ -193,6 +191,7 @@ public class TestRMFailover extends Clie
@Test
public void testWebAppProxyInStandAloneMode() throws YarnException,
InterruptedException, IOException {
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
WebAppProxyServer webAppProxyServer = new WebAppProxyServer();
try {
conf.set(YarnConfiguration.PROXY_ADDRESS, "0.0.0.0:9099");
@@ -227,6 +226,7 @@ public class TestRMFailover extends Clie
@Test
public void testEmbeddedWebAppProxy() throws YarnException,
InterruptedException, IOException {
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster.init(conf);
cluster.start();
getAdminService(0).transitionToActive(req);
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Fri Feb 7 18:42:06 2014
@@ -675,6 +675,7 @@ public class TestYarnCLI {
int result = spyCli.run(new String[] { "-help" });
Assert.assertTrue(result == 0);
verify(spyCli).printUsage(any(Options.class));
+ System.err.println(sysOutStream.toString()); //todo sandyt remove this hejfkdsl
Assert.assertEquals(createApplicationCLIHelpMessage(),
sysOutStream.toString());
@@ -748,6 +749,56 @@ public class TestYarnCLI {
"' doesn't exist in RM.", ex.getMessage());
}
}
+
+ @Test
+ public void testMoveApplicationAcrossQueues() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+
+ ApplicationReport newApplicationReport2 = ApplicationReport.newInstance(
+ applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+ newApplicationReport2);
+ int result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+ "-queue", "targetqueue"});
+ assertEquals(0, result);
+ verify(client, times(0)).moveApplicationAcrossQueues(
+ any(ApplicationId.class), any(String.class));
+ verify(sysOut).println(
+ "Application " + applicationId + " has already finished ");
+
+ ApplicationReport newApplicationReport = ApplicationReport.newInstance(
+ applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
+ "user", "queue", "appname", "host", 124, null,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
+ newApplicationReport);
+ result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+ "-queue", "targetqueue"});
+ assertEquals(0, result);
+ verify(client).moveApplicationAcrossQueues(any(ApplicationId.class),
+ any(String.class));
+ verify(sysOut).println("Moving application application_1234_0005 to queue targetqueue");
+ verify(sysOut).println("Successfully completed move.");
+
+ doThrow(new ApplicationNotFoundException("Application with id '"
+ + applicationId + "' doesn't exist in RM.")).when(client)
+ .moveApplicationAcrossQueues(applicationId, "targetqueue");
+ cli = createAndGetAppCLI();
+ try {
+ result = cli.run(new String[] { "-movetoqueue", applicationId.toString(),
+ "-queue", "targetqueue"});
+ Assert.fail();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof ApplicationNotFoundException);
+ Assert.assertEquals("Application with id '" + applicationId +
+ "' doesn't exist in RM.", ex.getMessage());
+ }
+ }
@Test
public void testListClusterNodes() throws Exception {
@@ -1087,23 +1138,28 @@ public class TestYarnCLI {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("usage: application");
- pw.println(" -appStates <States> Works with -list to filter applications based");
- pw.println(" on input comma-separated list of application");
- pw.println(" states. The valid application state can be one");
- pw.println(" of the following:");
- pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING,");
- pw.println(" FINISHED,FAILED,KILLED");
- pw.println(" -appTypes <Types> Works with -list to filter applications based");
- pw.println(" on input comma-separated list of application");
- pw.println(" types.");
- pw.println(" -help Displays help for all commands.");
- pw.println(" -kill <Application ID> Kills the application.");
- pw.println(" -list List applications from the RM. Supports");
- pw.println(" optional use of -appTypes to filter");
- pw.println(" applications based on application type, and");
- pw.println(" -appStates to filter applications based on");
- pw.println(" application state");
- pw.println(" -status <Application ID> Prints the status of the application.");
+ pw.println(" -appStates <States> Works with -list to filter applications");
+ pw.println(" based on input comma-separated list of");
+ pw.println(" application states. The valid application");
+ pw.println(" state can be one of the following:");
+ pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUN");
+ pw.println(" NING,FINISHED,FAILED,KILLED");
+ pw.println(" -appTypes <Types> Works with -list to filter applications");
+ pw.println(" based on input comma-separated list of");
+ pw.println(" application types.");
+ pw.println(" -help Displays help for all commands.");
+ pw.println(" -kill <Application ID> Kills the application.");
+ pw.println(" -list List applications from the RM. Supports");
+ pw.println(" optional use of -appTypes to filter");
+ pw.println(" applications based on application type,");
+ pw.println(" and -appStates to filter applications");
+ pw.println(" based on application state");
+ pw.println(" -movetoqueue <Application ID> Moves the application to a different");
+ pw.println(" queue.");
+ pw.println(" -queue <Queue Name> Works with the movetoqueue command to");
+ pw.println(" specify which queue to move an");
+ pw.println(" application to.");
+ pw.println(" -status <Application ID> Prints the status of the application.");
pw.close();
String appsHelpStr = baos.toString("UTF-8");
return appsHelpStr;
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Feb 7 18:42:06 2014
@@ -405,17 +405,20 @@
</property>
<property>
- <description>Enable automatic failover.</description>
+ <description>Enable automatic failover.
+ By default, it is enabled only when HA is enabled</description>
<name>yarn.resourcemanager.ha.automatic-failover.enabled</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
- <description>Enable embedded automatic failover. The embedded elector
- relies on the RM state store to handle fencing, and is primarily intended
- to be used in conjunction with ZKRMStateStore.</description>
+ <description>Enable embedded automatic failover.
+ By default, it is enabled only when HA is enabled.
+ The embedded elector relies on the RM state store to handle fencing,
+ and is primarily intended to be used in conjunction with ZKRMStateStore.
+ </description>
<name>yarn.resourcemanager.ha.automatic-failover.embedded</name>
- <value>false</value>
+ <value>true</value>
</property>
<property>
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestHAUtil.java Fri Feb 7 18:42:06 2014
@@ -159,8 +159,9 @@ public class TestHAUtil {
String confKey =
HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID);
assertEquals("YarnRuntimeException by Configuration#set()",
- HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(confKey),
- e.getMessage());
+ HAUtil.BAD_CONFIG_MESSAGE_PREFIX + HAUtil.getNeedToSetValueMessage(
+ HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME, RM1_NODE_ID)
+ + " or " + confKey), e.getMessage());
}
// simulate the case YarnConfiguration.RM_HA_IDS doesn't contain
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Fri Feb 7 18:42:06 2014
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.junit.Before;
import org.junit.Test;
@@ -119,6 +120,7 @@ public class TestRMHA {
*/
@Test (timeout = 30000)
public void testStartAndTransitions() throws IOException {
+ configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
Configuration conf = new YarnConfiguration(configuration);
rm = new MockRM(conf);
rm.init(conf);
@@ -178,7 +180,6 @@ public class TestRMHA {
"automatic failover is enabled";
Configuration conf = new YarnConfiguration(configuration);
- conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, true);
rm = new MockRM(conf);
rm.init(conf);
@@ -236,6 +237,7 @@ public class TestRMHA {
String errorMessageForEventHandler =
"Expect to get the same number of handlers";
String errorMessageForService = "Expect to get the same number of services";
+ configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
Configuration conf = new YarnConfiguration(configuration);
rm = new MockRM(conf) {
@Override
@@ -313,6 +315,57 @@ public class TestRMHA {
}
}
+ @Test
+ public void testHAWithRMHostName() {
+ //test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set
+ //We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration
+ configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+ RM1_NODE_ID), "1.1.1.1");
+ configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+ RM2_NODE_ID), "0.0.0.0");
+ configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+ RM3_NODE_ID), "2.2.2.2");
+ try {
+ Configuration conf = new YarnConfiguration(configuration);
+ rm = new MockRM(conf);
+ rm.init(conf);
+ for (String confKey : YarnConfiguration.RM_SERVICES_ADDRESS_CONF_KEYS) {
+ assertEquals("RPC address not set for " + confKey,
+ RM1_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM1_NODE_ID)));
+ assertEquals("RPC address not set for " + confKey,
+ RM2_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM2_NODE_ID)));
+ assertEquals("RPC address not set for " + confKey,
+ RM3_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM3_NODE_ID)));
+ }
+ } catch (YarnRuntimeException e) {
+ fail("Should not throw any exceptions.");
+ }
+
+ //test if only RM_HOSTBANE_{rm_id} is set
+ configuration.clear();
+ configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+ + RM2_NODE_ID);
+ configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+ RM1_NODE_ID), "1.1.1.1");
+ configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
+ RM2_NODE_ID), "0.0.0.0");
+ try {
+ Configuration conf = new YarnConfiguration(configuration);
+ rm = new MockRM(conf);
+ rm.init(conf);
+ assertEquals("RPC address not set for " + YarnConfiguration.RM_ADDRESS,
+ "1.1.1.1:8032",
+ conf.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM1_NODE_ID)));
+ assertEquals("RPC address not set for " + YarnConfiguration.RM_ADDRESS,
+ "0.0.0.0:8032",
+ conf.get(HAUtil.addSuffix(YarnConfiguration.RM_ADDRESS, RM2_NODE_ID)));
+
+ } catch (YarnRuntimeException e) {
+ fail("Should not throw any exceptions.");
+ }
+ }
+
@SuppressWarnings("rawtypes")
class MyCountingDispatcher extends AbstractService implements Dispatcher {
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Fri Feb 7 18:42:06 2014
@@ -159,6 +159,7 @@ public class TestZKRMStateStore extends
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
Configuration conf1 = createHARMConf("rm1,rm2", "rm1", 1234);
+ conf1.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
ResourceManager rm1 = new ResourceManager();
rm1.init(conf1);
rm1.start();
@@ -170,6 +171,7 @@ public class TestZKRMStateStore extends
rm1.getRMContext().getRMAdminService().getServiceStatus().getState());
Configuration conf2 = createHARMConf("rm1,rm2", "rm2", 5678);
+ conf2.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
ResourceManager rm2 = new ResourceManager();
rm2.init(conf2);
rm2.start();
Modified: hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java?rev=1565757&r1=1565756&r2=1565757&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java (original)
+++ hadoop/common/branches/HDFS-5698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYARNClusterForHA.java Fri Feb 7 18:42:06 2014
@@ -42,7 +42,7 @@ public class TestMiniYARNClusterForHA {
@Before
public void setup() throws IOException, InterruptedException {
Configuration conf = new YarnConfiguration();
-
+ conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster = new MiniYARNCluster(TestMiniYARNClusterForHA.class.getName(),
2, 1, 1, 1);
cluster.init(conf);