You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2015/12/23 00:51:46 UTC
[47/50] [abbrv] hadoop git commit: YARN-4290. Add -showDetails option
to YARN Nodes CLI to print all nodes reports information. (Sunil G via
wangda)
YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports information. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0de7028
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0de7028
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0de7028
Branch: refs/heads/yarn-2877
Commit: a0de7028515eebe1c526cc42808cdbc8ed6b4e2a
Parents: bc038b3
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Dec 21 11:36:22 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Dec 21 11:36:22 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 76 +++++++++++++++++++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 45 ++++++++++--
3 files changed, 115 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0de7028/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f575003..10f54e6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -634,6 +634,9 @@ Release 2.8.0 - UNRELEASED
YARN-4480. Clean up some inappropriate imports. (Kai Zheng via umamahesh)
+ YARN-4290. Add -showDetails option to YARN Nodes CLI to print all nodes reports
+ information. (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0de7028/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index c93b2df..c112857 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -54,6 +54,7 @@ public class NodeCLI extends YarnCLI {
private static final String NODE_STATE_CMD = "states";
private static final String NODE_ALL = "all";
+ private static final String NODE_SHOW_DETAILS = "showDetails";
public static void main(String[] args) throws Exception {
NodeCLI cli = new NodeCLI();
@@ -72,7 +73,8 @@ public class NodeCLI extends YarnCLI {
opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
opts.addOption(LIST_CMD, false, "List all running nodes. " +
"Supports optional use of -states to filter nodes " +
- "based on node state, all -all to list all nodes.");
+ "based on node state, all -all to list all nodes, " +
+ "-showDetails to display more details about each node.");
Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
"Works with -list to filter nodes based on input comma-separated list of node states.");
nodeStateOpt.setValueSeparator(',');
@@ -82,6 +84,9 @@ public class NodeCLI extends YarnCLI {
Option allOpt = new Option(NODE_ALL, false,
"Works with -list to list all nodes.");
opts.addOption(allOpt);
+ Option showDetailsOpt = new Option(NODE_SHOW_DETAILS, false,
+ "Works with -list to show more details about each node.");
+ opts.addOption(showDetailsOpt);
opts.getOption(STATUS_CMD).setArgName("NodeId");
int exitCode = -1;
@@ -119,7 +124,13 @@ public class NodeCLI extends YarnCLI {
} else {
nodeStates.add(NodeState.RUNNING);
}
- listClusterNodes(nodeStates);
+
+ // List all node details with more information.
+ if (cliParser.hasOption(NODE_SHOW_DETAILS)) {
+ listDetailedClusterNodes(nodeStates);
+ } else {
+ listClusterNodes(nodeStates);
+ }
} else if (cliParser.hasOption(HELP_CMD)) {
printUsage(opts);
return 0;
@@ -164,6 +175,67 @@ public class NodeCLI extends YarnCLI {
}
/**
+ * Lists the nodes which are matching the given node states along with
+ * detailed node informations such as resource usage etc.
+ *
+ * @param nodeStates
+ * @throws YarnException
+ * @throws IOException
+ */
+ private void listDetailedClusterNodes(Set<NodeState> nodeStates)
+ throws YarnException, IOException {
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(sysout,
+ Charset.forName("UTF-8")));
+ List<NodeReport> nodesReport = client.getNodeReports(nodeStates
+ .toArray(new NodeState[0]));
+ writer.println("Total Nodes:" + nodesReport.size());
+ writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
+ "Number-of-Running-Containers");
+ for (NodeReport nodeReport : nodesReport) {
+ writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
+ nodeReport.getNodeState(), nodeReport.getHttpAddress(),
+ nodeReport.getNumContainers());
+ writer.println("Detailed Node Information :");
+ writer.print("\tConfigured Resources : ");
+ writer.println(nodeReport.getCapability());
+ writer.print("\tAllocated Resources : ");
+ if (nodeReport.getUsed() != null) {
+ writer.print(nodeReport.getUsed());
+ }
+ writer.println();
+
+ writer.print("\tResource Utilization by Node : ");
+ if (nodeReport.getNodeUtilization() != null) {
+ writer.print("PMem:"
+ + nodeReport.getNodeUtilization().getPhysicalMemory()
+ + " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory()
+ + " MB, VCores:" + nodeReport.getNodeUtilization().getCPU());
+ }
+ writer.println();
+
+ writer.print("\tResource Utilization by Containers : ");
+ if (nodeReport.getAggregatedContainersUtilization() != null) {
+ writer.print("PMem:"
+ + nodeReport.getAggregatedContainersUtilization()
+ .getPhysicalMemory()
+ + " MB, VMem:"
+ + nodeReport.getAggregatedContainersUtilization()
+ .getVirtualMemory() + " MB, VCores:"
+ + nodeReport.getAggregatedContainersUtilization().getCPU());
+ }
+ writer.println();
+
+ writer.print("\tNode-Labels : ");
+ // Create a List for node labels since we need it get sorted
+ List<String> nodeLabelsList = new ArrayList<String>(
+ nodeReport.getNodeLabels());
+ Collections.sort(nodeLabelsList);
+ writer.println(StringUtils.join(nodeLabelsList.iterator(), ','));
+ }
+ writer.flush();
+ }
+
+ /**
* Prints the node report for node id.
*
* @param nodeIdStr
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0de7028/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index f30bd40..1a77c7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -993,6 +993,35 @@ public class TestYarnCLI {
verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
+ result = cli.run(new String[] { "-list", "-showDetails" });
+ assertEquals(0, result);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:2");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Number-of-Running-Containers");
+ pw.print(" host0:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.println("Detailed Node Information :");
+ pw.println("\tConfigured Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
+ pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
+ pw.println("\tNode-Labels : ");
+ pw.print(" host1:0\t RUNNING\t host1:8888\t");
+ pw.println(" 0");
+ pw.println("Detailed Node Information :");
+ pw.println("\tConfigured Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
+ pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
+ pw.println("\tNode-Labels : ");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
nodeStates.clear();
nodeStates.add(NodeState.UNHEALTHY);
states = nodeStates.toArray(new NodeState[0]);
@@ -1011,7 +1040,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
- verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@@ -1032,7 +1061,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
- verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@@ -1053,7 +1082,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
- verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@@ -1074,7 +1103,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
- verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@@ -1107,7 +1136,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
- verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
sysOutStream.reset();
nodeStates.clear();
@@ -1142,7 +1171,7 @@ public class TestYarnCLI {
pw.close();
nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
- verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(10)).write(any(byte[].class), anyInt(), anyInt());
}
private List<NodeReport> getNodeReports(
@@ -1727,7 +1756,9 @@ public class TestYarnCLI {
pw.println(" -help Displays help for all commands.");
pw.println(" -list List all running nodes. Supports optional use of");
pw.println(" -states to filter nodes based on node state, all -all");
- pw.println(" to list all nodes.");
+ pw.println(" to list all nodes, -showDetails to display more");
+ pw.println(" details about each node.");
+ pw.println(" -showDetails Works with -list to show more details about each node.");
pw.println(" -states <States> Works with -list to filter nodes based on input");
pw.println(" comma-separated list of node states.");
pw.println(" -status <NodeId> Prints the status report of the node.");