You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/11/07 05:49:20 UTC
hadoop git commit: YARN-2647. Added a queue CLI for getting queue
information. Contributed by Sunil Govind.
Repository: hadoop
Updated Branches:
refs/heads/trunk ba0a42cf6 -> 61effcbeb
YARN-2647. Added a queue CLI for getting queue information. Contributed by Sunil Govind.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61effcbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61effcbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61effcbe
Branch: refs/heads/trunk
Commit: 61effcbebc819ee1bd3d60f5444c39c972494a32
Parents: ba0a42c
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Nov 6 20:48:31 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Nov 6 20:48:31 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 +
.../apache/hadoop/yarn/client/cli/QueueCLI.java | 148 +++++++++++++++++++
.../hadoop/yarn/client/cli/TestYarnCLI.java | 61 ++++++++
5 files changed, 224 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61effcbe/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f6b39e3..9dffb22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.6.0 - UNRELEASED
YARN-2198. Remove the need to run NodeManager as privileged account for
Windows Secure Container Executor. (Remus Rusanu via jianhe)
+ YARN-2647. Added a queue CLI for getting queue information. (Sunil Govind via
+ vinodkv)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61effcbe/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index bfcb190..0d83674 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -30,6 +30,7 @@ function hadoop_usage
echo " node prints node report(s)"
echo " nodemanager run a nodemanager on each slave"
echo " proxyserver run the web app proxy server"
+ echo " queue prints queue information"
echo " resourcemanager run the ResourceManager"
echo " resourcemanager -format-state-store deletes the RMStateStore"
echo " rmadmin admin tools"
@@ -127,6 +128,11 @@ case "${COMMAND}" in
JAVA_HEAP_MAX="-Xmx${YARN_PROXYSERVER_HEAPSIZE}m"
fi
;;
+ queue)
+ CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
resourcemanager)
daemon="true"
CLASS='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61effcbe/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 0cf0b40..7b7fdc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -196,6 +196,11 @@ goto :eof
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
goto :eof
+:queue
+ set CLASS=org.apache.hadoop.yarn.client.cli.QueueCLI
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
:resourcemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\rm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.resourcemanager.ResourceManager
@@ -298,6 +303,7 @@ goto :eof
@echo applicationattempt prints applicationattempt(s) report
@echo container prints container(s) report
@echo node prints node report(s)
+ @echo queue prints queue information
@echo logs dump container logs
@echo classpath prints the class path needed to get the
@echo Hadoop jar and the required libraries
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61effcbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
new file mode 100644
index 0000000..3c74f13
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.cli;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Options;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import com.google.common.annotations.VisibleForTesting;
+
+@Private
+@Unstable
+public class QueueCLI extends YarnCLI {
+ public static final String QUEUE = "queue";
+
+ public static void main(String[] args) throws Exception {
+ QueueCLI cli = new QueueCLI();
+ cli.setSysOutPrintStream(System.out);
+ cli.setSysErrPrintStream(System.err);
+ int res = ToolRunner.run(cli, args);
+ cli.stop();
+ System.exit(res);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options opts = new Options();
+ int exitCode = -1;
+ if (args.length > 0) {
+ opts.addOption(STATUS_CMD, true,
+ "List queue information about given queue.");
+ opts.addOption(HELP_CMD, false, "Displays help for all commands.");
+ opts.getOption(STATUS_CMD).setArgName("Queue Name");
+ } else {
+ syserr.println("Invalid Command usage. Command must start with 'queue'");
+ return exitCode;
+ }
+
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ sysout.println("Missing argument for options");
+ printUsage(opts);
+ return exitCode;
+ }
+
+ if (cliParser.hasOption(STATUS_CMD)) {
+ if (args.length != 3) {
+ printUsage(opts);
+ return exitCode;
+ }
+ listQueue(cliParser.getOptionValue(STATUS_CMD));
+ } else if (cliParser.hasOption(HELP_CMD)) {
+ printUsage(opts);
+ return 0;
+ } else {
+ syserr.println("Invalid Command Usage : ");
+ printUsage(opts);
+ }
+ return 0;
+ }
+
+ /**
+ * It prints the usage of the command
+ *
+ * @param opts
+ */
+ @VisibleForTesting
+ void printUsage(Options opts) {
+ new HelpFormatter().printHelp(QUEUE, opts);
+ }
+
+ /**
+ * Lists the Queue Information matching the given queue name
+ *
+ * @param queueName
+ * @throws YarnException
+ * @throws IOException
+ */
+ private void listQueue(String queueName) throws YarnException, IOException {
+ PrintWriter writer = new PrintWriter(sysout);
+
+ QueueInfo queueInfo = client.getQueueInfo(queueName);
+ writer.println("Queue Information : ");
+ printQueueInfo(writer, queueInfo);
+ writer.flush();
+ }
+
+ private void printQueueInfo(PrintWriter writer, QueueInfo queueInfo) {
+ writer.print("Queue Name : ");
+ writer.println(queueInfo.getQueueName());
+
+ writer.print("\tState : ");
+ writer.println(queueInfo.getQueueState());
+ DecimalFormat df = new DecimalFormat("#.0");
+ writer.print("\tCapacity : ");
+ writer.println(df.format(queueInfo.getCapacity() * 100) + "%");
+ writer.print("\tCurrent Capacity : ");
+ writer.println(df.format(queueInfo.getCurrentCapacity() * 100) + "%");
+ writer.print("\tMaximum Capacity : ");
+ writer.println(df.format(queueInfo.getMaximumCapacity() * 100) + "%");
+ writer.print("\tDefault Node Label expression : ");
+ if (null != queueInfo.getDefaultNodeLabelExpression()) {
+ writer.println(queueInfo.getDefaultNodeLabelExpression());
+ } else {
+ writer.println();
+ }
+
+ Set<String> nodeLabels = queueInfo.getAccessibleNodeLabels();
+ StringBuilder labelList = new StringBuilder();
+ writer.print("\tAccessible Node Labels : ");
+ for (String nodeLabel : nodeLabels) {
+ if (labelList.length() > 0) {
+ labelList.append(',');
+ }
+ labelList.append(nodeLabel);
+ }
+ writer.println(labelList.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61effcbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 19ded9c..adb58e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -1235,6 +1237,57 @@ public class TestYarnCLI {
Assert.assertEquals(String.format("Missing argument for options%n%1s",
createNodeCLIHelpMessage()), sysOutStream.toString());
}
+
+ @Test
+ public void testGetQueueInfo() throws Exception {
+ QueueCLI cli = createAndGetQueueCLI();
+ Set<String> nodeLabels = new HashSet<String>();
+ nodeLabels.add("GPU");
+ nodeLabels.add("JDK_7");
+ QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
+ null, null, QueueState.RUNNING, nodeLabels, "GPU");
+ when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
+ int result = cli.run(new String[] { "queue", "-status", "queueA" });
+ assertEquals(0, result);
+ verify(client).getQueueInfo("queueA");
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Queue Information : ");
+ pw.println("Queue Name : " + "queueA");
+ pw.println("\tState : " + "RUNNING");
+ pw.println("\tCapacity : " + "40.0%");
+ pw.println("\tCurrent Capacity : " + "50.0%");
+ pw.println("\tMaximum Capacity : " + "80.0%");
+ pw.println("\tDefault Node Label expression : " + "GPU");
+ pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
+ pw.close();
+ String queueInfoStr = baos.toString("UTF-8");
+ Assert.assertEquals(queueInfoStr, sysOutStream.toString());
+ }
+
+ @Test
+ public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
+ QueueCLI cli = createAndGetQueueCLI();
+ QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
+ null, null, QueueState.RUNNING, null, null);
+ when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
+ int result = cli.run(new String[] { "queue", "-status", "queueA" });
+ assertEquals(0, result);
+ verify(client).getQueueInfo("queueA");
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Queue Information : ");
+ pw.println("Queue Name : " + "queueA");
+ pw.println("\tState : " + "RUNNING");
+ pw.println("\tCapacity : " + "40.0%");
+ pw.println("\tCurrent Capacity : " + "50.0%");
+ pw.println("\tMaximum Capacity : " + "80.0%");
+ pw.println("\tDefault Node Label expression : ");
+ pw.println("\tAccessible Node Labels : ");
+ pw.close();
+ String queueInfoStr = baos.toString("UTF-8");
+ Assert.assertEquals(queueInfoStr, sysOutStream.toString());
+ }
private void verifyUsageInfo(YarnCLI cli) throws Exception {
cli.setSysErrPrintStream(sysErr);
@@ -1272,6 +1325,14 @@ public class TestYarnCLI {
cli.setSysOutPrintStream(sysOut);
return cli;
}
+
+ private QueueCLI createAndGetQueueCLI() {
+ QueueCLI cli = new QueueCLI();
+ cli.setClient(client);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+ return cli;
+ }
private String createApplicationCLIHelpMessage() throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();