You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/03/09 21:25:57 UTC
[31/50] [abbrv] hadoop git commit: YARN-2786. Created a yarn cluster
CLI and seeded with one command for listing node-labels collection.
Contributed by Wangda Tan.
YARN-2786. Created a yarn cluster CLI and seeded with one command for listing node-labels collection. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a638ed6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a638ed6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a638ed6
Branch: refs/heads/HDFS-7285
Commit: 7a638ed67a225ea45b258db6926f51bb354c1564
Parents: 22b1f53
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Mar 5 10:54:34 2015 -0800
Committer: Jing Zhao <ji...@apache.org>
Committed: Mon Mar 9 13:11:25 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 6 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 6 +
.../hadoop/yarn/client/cli/ClusterCLI.java | 157 ++++++++++++++++++
.../hadoop/yarn/client/cli/TestClusterCLI.java | 158 +++++++++++++++++++
5 files changed, 330 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a638ed6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5f61462..dcf328f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -97,6 +97,9 @@ Release 2.7.0 - UNRELEASED
YARN-3249. Add a 'kill application' button to Resource Manager's Web UI.
(Ryu Kobayashi via ozawa)
+ YARN-2786. Created a yarn cluster CLI and seeded with one command for listing
+ node-labels collection. (Wangda Tan via vinodkv)
+
IMPROVEMENTS
YARN-3005. [JDK7] Use switch statement for String instead of if-else
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a638ed6/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index f1a06a6..e6af4ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -25,6 +25,7 @@ function hadoop_usage
echo " applicationattempt prints applicationattempt(s) report"
echo " classpath prints the class path needed to get the"
echo " Hadoop jar and the required libraries"
+ echo " cluster prints cluster information"
echo " container prints container(s) report"
echo " daemonlog get/set the log level for each daemon"
echo " jar <jar> run a jar file"
@@ -83,6 +84,11 @@ case "${COMMAND}" in
classpath)
hadoop_do_classpath_subcommand "$@"
;;
+ cluster)
+ CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
+ hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
+ YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ ;;
daemonlog)
CLASS=org.apache.hadoop.log.LogLevel
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a638ed6/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 3f68b16..c29ee53 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -192,6 +192,11 @@ goto :eof
set yarn-command-arguments=%yarn-command% %yarn-command-arguments%
goto :eof
+:cluster
+ set CLASS=org.apache.hadoop.yarn.client.cli.ClusterCLI
+ set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
+ goto :eof
+
:container
set CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
set YARN_OPTS=%YARN_OPTS% %YARN_CLIENT_OPTS%
@@ -312,6 +317,7 @@ goto :eof
@echo jar ^<jar^> run a jar file
@echo application prints application(s) report/kill application
@echo applicationattempt prints applicationattempt(s) report
+ @echo cluster prints cluster information
@echo container prints container(s) report
@echo node prints node report(s)
@echo queue prints queue information
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a638ed6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java
new file mode 100644
index 0000000..3924803
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ClusterCLI.java
@@ -0,0 +1,157 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Cluster CLI used to get over all information of the cluster
+ */
+@Private
+public class ClusterCLI extends YarnCLI {
+ private static final String TITLE = "yarn cluster";
+ public static final String LIST_LABELS_CMD = "list-node-labels";
+ public static final String DIRECTLY_ACCESS_NODE_LABEL_STORE =
+ "directly-access-node-label-store";
+ public static final String CMD = "cluster";
+ private boolean accessLocal = false;
+ static CommonNodeLabelsManager localNodeLabelsManager = null;
+
+ public static void main(String[] args) throws Exception {
+ ClusterCLI cli = new ClusterCLI();
+ cli.setSysOutPrintStream(System.out);
+ cli.setSysErrPrintStream(System.err);
+ int res = ToolRunner.run(cli, args);
+ cli.stop();
+ System.exit(res);
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ Options opts = new Options();
+
+ opts.addOption("lnl", LIST_LABELS_CMD, false,
+ "List cluster node-label collection");
+ opts.addOption("h", HELP_CMD, false, "Displays help for all commands.");
+ opts.addOption("dnl", DIRECTLY_ACCESS_NODE_LABEL_STORE, false,
+ "Directly access node label store, "
+ + "with this option, all node label related operations"
+ + " will NOT connect RM. Instead, they will"
+ + " access/modify stored node labels directly."
+ + " By default, it is false (access via RM)."
+ + " AND PLEASE NOTE: if you configured "
+ + YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR
+ + " to a local directory"
+ + " (instead of NFS or HDFS), this option will only work"
+ + " when the command run on the machine where RM is running."
+ + " Also, this option is UNSTABLE, could be removed in future"
+ + " releases.");
+
+ int exitCode = -1;
+ CommandLine parsedCli = null;
+ try {
+ parsedCli = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ sysout.println("Missing argument for options");
+ printUsage(opts);
+ return exitCode;
+ }
+
+ if (parsedCli.hasOption(DIRECTLY_ACCESS_NODE_LABEL_STORE)) {
+ accessLocal = true;
+ }
+
+ if (parsedCli.hasOption(LIST_LABELS_CMD)) {
+ printClusterNodeLabels();
+ } else if (parsedCli.hasOption(HELP_CMD)) {
+ printUsage(opts);
+ return 0;
+ } else {
+ syserr.println("Invalid Command Usage : ");
+ printUsage(opts);
+ }
+ return 0;
+ }
+
+ private List<String> sortStrSet(Set<String> labels) {
+ List<String> list = new ArrayList<String>();
+ list.addAll(labels);
+ Collections.sort(list);
+ return list;
+ }
+
+ void printClusterNodeLabels() throws YarnException, IOException {
+ Set<String> nodeLabels = null;
+ if (accessLocal) {
+ nodeLabels =
+ getNodeLabelManagerInstance(getConf()).getClusterNodeLabels();
+ } else {
+ nodeLabels = client.getClusterNodeLabels();
+ }
+ sysout.println(String.format("Node Labels: %s",
+ StringUtils.join(sortStrSet(nodeLabels).iterator(), ",")));
+ }
+
+ @VisibleForTesting
+ static synchronized CommonNodeLabelsManager
+ getNodeLabelManagerInstance(Configuration conf) {
+ if (localNodeLabelsManager == null) {
+ localNodeLabelsManager = new CommonNodeLabelsManager();
+ localNodeLabelsManager.init(conf);
+ localNodeLabelsManager.start();
+ }
+ return localNodeLabelsManager;
+ }
+
+ @VisibleForTesting
+ void printUsage(Options opts) throws UnsupportedEncodingException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw =
+ new PrintWriter(new OutputStreamWriter(baos, Charset.forName("UTF-8")));
+ new HelpFormatter().printHelp(pw, HelpFormatter.DEFAULT_WIDTH, TITLE, null,
+ opts, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD,
+ null);
+ pw.close();
+ sysout.println(baos.toString("UTF-8"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a638ed6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
new file mode 100644
index 0000000..f9ccf87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestClusterCLI.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.HashSet;
+
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+public class TestClusterCLI {
+ ByteArrayOutputStream sysOutStream;
+ private PrintStream sysOut;
+ ByteArrayOutputStream sysErrStream;
+ private PrintStream sysErr;
+
+ @Before
+ public void setup() {
+ sysOutStream = new ByteArrayOutputStream();
+ sysOut = spy(new PrintStream(sysOutStream));
+ sysErrStream = new ByteArrayOutputStream();
+ sysErr = spy(new PrintStream(sysErrStream));
+ System.setOut(sysOut);
+ }
+
+ @Test
+ public void testGetClusterNodeLabels() throws Exception {
+ YarnClient client = mock(YarnClient.class);
+ when(client.getClusterNodeLabels()).thenReturn(
+ ImmutableSet.of("label1", "label2"));
+ ClusterCLI cli = new ClusterCLI();
+ cli.setClient(client);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+
+ int rc =
+ cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
+ assertEquals(0, rc);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.print("Node Labels: label1,label2");
+ pw.close();
+ verify(sysOut).println(baos.toString("UTF-8"));
+ }
+
+ @Test
+ public void testGetClusterNodeLabelsWithLocalAccess() throws Exception {
+ YarnClient client = mock(YarnClient.class);
+ when(client.getClusterNodeLabels()).thenReturn(
+ ImmutableSet.of("remote1", "remote2"));
+ ClusterCLI cli = new ClusterCLI();
+ cli.setClient(client);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+ ClusterCLI.localNodeLabelsManager = mock(CommonNodeLabelsManager.class);
+ when(ClusterCLI.localNodeLabelsManager.getClusterNodeLabels())
+ .thenReturn(ImmutableSet.of("local1", "local2"));
+
+ int rc =
+ cli.run(new String[] { ClusterCLI.CMD,
+ "-" + ClusterCLI.LIST_LABELS_CMD,
+ "-" + ClusterCLI.DIRECTLY_ACCESS_NODE_LABEL_STORE });
+ assertEquals(0, rc);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ // it should return local* instead of remote*
+ pw.print("Node Labels: local1,local2");
+ pw.close();
+ verify(sysOut).println(baos.toString("UTF-8"));
+ }
+
+ @Test
+ public void testGetEmptyClusterNodeLabels() throws Exception {
+ YarnClient client = mock(YarnClient.class);
+ when(client.getClusterNodeLabels()).thenReturn(new HashSet<String>());
+ ClusterCLI cli = new ClusterCLI();
+ cli.setClient(client);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+
+ int rc =
+ cli.run(new String[] { ClusterCLI.CMD, "-" + ClusterCLI.LIST_LABELS_CMD });
+ assertEquals(0, rc);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.print("Node Labels: ");
+ pw.close();
+ verify(sysOut).println(baos.toString("UTF-8"));
+ }
+
+ @Test
+ public void testHelp() throws Exception {
+ ClusterCLI cli = new ClusterCLI();
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+
+ int rc =
+ cli.run(new String[] { "cluster", "--help" });
+ assertEquals(0, rc);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("usage: yarn cluster");
+ pw.println(" -dnl,--directly-access-node-label-store Directly access node label");
+ pw.println(" store, with this option, all");
+ pw.println(" node label related operations");
+ pw.println(" will NOT connect RM. Instead,");
+ pw.println(" they will access/modify stored");
+ pw.println(" node labels directly. By");
+ pw.println(" default, it is false (access");
+ pw.println(" via RM). AND PLEASE NOTE: if");
+ pw.println(" you configured");
+ pw.println(" yarn.node-labels.fs-store.root-");
+ pw.println(" dir to a local directory");
+ pw.println(" (instead of NFS or HDFS), this");
+ pw.println(" option will only work when the");
+ pw.println(" command run on the machine");
+ pw.println(" where RM is running. Also, this");
+ pw.println(" option is UNSTABLE, could be");
+ pw.println(" removed in future releases.");
+ pw.println(" -h,--help Displays help for all commands.");
+ pw.println(" -lnl,--list-node-labels List cluster node-label");
+ pw.println(" collection");
+ pw.close();
+ verify(sysOut).println(baos.toString("UTF-8"));
+ }
+}
\ No newline at end of file