You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/08/27 19:05:15 UTC
svn commit: r1517887 [1/2] - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoo...
Author: arp
Date: Tue Aug 27 17:05:13 2013
New Revision: 1517887
URL: http://svn.apache.org/r1517887
Log:
Merge all changes from trunk to branch HDFS-2832.
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Tue Aug 27 17:05:13 2013
@@ -23,6 +23,8 @@ Release 2.3.0 - UNRELEASED
IMPROVEMENTS
+ YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -42,7 +44,16 @@ Release 2.1.1-beta - UNRELEASED
IMPROVEMENTS
YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
+
+ YARN-1074. Cleaned up YARN CLI application list to only display running
+ applications by default. (Xuan Gong via vinodkv)
+ YARN-1093. Corrections to Fair Scheduler documentation (Wing Yew Poon via
+ Sandy Ryza)
+
+ YARN-942. In Fair Scheduler documentation, inconsistency on which
+ properties have prefix (Akira Ajisaka via Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
@@ -85,6 +96,15 @@ Release 2.1.1-beta - UNRELEASED
YARN-1082. Create base directories on HDFS after RM login to ensure RM
recovery doesn't fail in secure mode. (vinodkv via acmurthy)
+ YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in
+ secure setup with kerberos. (Omkar Vinit Joshi via vinodkv)
+
+ YARN-1094. Fixed a blocker with RM restart code because of which RM crashes
+ when try to recover an existing app. (vinodkv)
+
+ YARN-1008. MiniYARNCluster with multiple nodemanagers, all nodes have same
+ key for allocations. (tucu)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetApplicationsRequest.java Tue Aug 27 17:05:13 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.protocolrecords;
+import java.util.EnumSet;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -25,6 +26,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -45,13 +47,65 @@ public abstract class GetApplicationsReq
return request;
}
+ /**
+ * <p>
+ * The request from clients to get a report of Applications matching the
+ * giving application types in the cluster from the
+ * <code>ResourceManager</code>.
+ * </p>
+ *
+ *
+ * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ */
+ @Public
+ @Stable
+ public static GetApplicationsRequest
+ newInstance(Set<String> applicationTypes) {
+ GetApplicationsRequest request =
+ Records.newRecord(GetApplicationsRequest.class);
+ request.setApplicationTypes(applicationTypes);
+ return request;
+ }
+
+ /**
+ * <p>
+ * The request from clients to get a report of Applications matching the
+ * giving application states in the cluster from the
+ * <code>ResourceManager</code>.
+ * </p>
+ *
+ *
+ * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ */
@Public
@Stable
public static GetApplicationsRequest newInstance(
- Set<String> applicationTypes) {
+ EnumSet<YarnApplicationState> applicationStates) {
+ GetApplicationsRequest request =
+ Records.newRecord(GetApplicationsRequest.class);
+ request.setApplicationStates(applicationStates);
+ return request;
+ }
+
+ /**
+ * <p>
+ * The request from clients to get a report of Applications matching the
+ * giving and application types and application types in the cluster from the
+ * <code>ResourceManager</code>.
+ * </p>
+ *
+ *
+ * @see ApplicationClientProtocol#getApplications(GetApplicationsRequest)
+ */
+ @Public
+ @Stable
+ public static GetApplicationsRequest newInstance(
+ Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates) {
GetApplicationsRequest request =
Records.newRecord(GetApplicationsRequest.class);
request.setApplicationTypes(applicationTypes);
+ request.setApplicationStates(applicationStates);
return request;
}
@@ -75,4 +129,25 @@ public abstract class GetApplicationsReq
@Unstable
public abstract void
setApplicationTypes(Set<String> applicationTypes);
+
+ /**
+ * Get the application states to filter applications on
+ *
+ * @return Set of Application states to filter on
+ */
+ @Public
+ @Stable
+ public abstract EnumSet<YarnApplicationState> getApplicationStates();
+
+ /**
+ * Set the application states to filter applications on
+ *
+ * @param applicationStates
+ * A Set of Application states to filter on.
+ * If not defined, match all running applications
+ */
+ @Private
+ @Unstable
+ public abstract void
+ setApplicationStates(EnumSet<YarnApplicationState> applicationStates);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/GetDelegationTokenRequest.java Tue Aug 27 17:05:13 2013
@@ -33,7 +33,7 @@ public abstract class GetDelegationToken
@Public
@Stable
- public GetDelegationTokenRequest newInstance(String renewer) {
+ public static GetDelegationTokenRequest newInstance(String renewer) {
GetDelegationTokenRequest request =
Records.newRecord(GetDelegationTokenRequest.class);
request.setRenewer(renewer);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Tue Aug 27 17:05:13 2013
@@ -132,6 +132,15 @@ public class YarnConfiguration extends C
RM_PREFIX + "scheduler.client.thread-count";
public static final int DEFAULT_RM_SCHEDULER_CLIENT_THREAD_COUNT = 50;
+ /** If the port should be included or not in the node name. The node name
+ * is used by the scheduler for resource requests allocation location
+ * matching. Typically this is just the hostname, using the port is needed
+ * when using minicluster and specific NM are required.*/
+ public static final String RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME =
+ YARN_PREFIX + "scheduler.include-port-in-node-name";
+ public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME =
+ false;
+
/**
* Enable periodic monitor threads.
* @see #RM_SCHEDULER_MONITOR_POLICIES
@@ -205,6 +214,14 @@ public class YarnConfiguration extends C
public static final String RM_KEYTAB =
RM_PREFIX + "keytab";
+ /**The kerberos principal to be used for spnego filter for RM.*/
+ public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
+ RM_PREFIX + "webapp.spnego-principal";
+
+ /**The kerberos keytab to be used for spnego filter for RM.*/
+ public static final String RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+ RM_PREFIX + "webapp.spnego-keytab-file";
+
/** How long to wait until a container is considered dead.*/
public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS =
RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
@@ -599,7 +616,15 @@ public class YarnConfiguration extends C
public static final String NM_USER_HOME_DIR =
NM_PREFIX + "user-home-dir";
-
+
+ /**The kerberos principal to be used for spnego filter for NM.*/
+ public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
+ NM_PREFIX + "webapp.spnego-principal";
+
+ /**The kerberos keytab to be used for spnego filter for NM.*/
+ public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
+ NM_PREFIX + "webapp.spnego-keytab-file";
+
public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
////////////////////////////////
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Tue Aug 27 17:05:13 2013
@@ -122,6 +122,7 @@ message GetClusterMetricsResponseProto {
message GetApplicationsRequestProto {
repeated string application_types = 1;
+ repeated YarnApplicationStateProto application_states = 2;
}
message GetApplicationsResponseProto {
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java Tue Aug 27 17:05:13 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api;
import java.io.IOException;
+import java.util.EnumSet;
import java.util.List;
import java.util.Set;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Token;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -171,13 +173,13 @@ public abstract class YarnClient extends
* <p>
* Get a report (ApplicationReport) of all Applications in the cluster.
* </p>
- *
+ *
* <p>
* If the user does not have <code>VIEW_APP</code> access for an application
* then the corresponding report will be filtered as described in
* {@link #getApplicationReport(ApplicationId)}.
* </p>
- *
+ *
* @return a list of reports of all running applications
* @throws YarnException
* @throws IOException
@@ -207,6 +209,50 @@ public abstract class YarnClient extends
/**
* <p>
+ * Get a report (ApplicationReport) of Applications matching the given
+ * application states in the cluster.
+ * </p>
+ *
+ * <p>
+ * If the user does not have <code>VIEW_APP</code> access for an application
+ * then the corresponding report will be filtered as described in
+ * {@link #getApplicationReport(ApplicationId)}.
+ * </p>
+ *
+ * @param applicationStates
+ * @return a list of reports of applications
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract List<ApplicationReport>
+ getApplications(EnumSet<YarnApplicationState> applicationStates)
+ throws YarnException, IOException;
+
+ /**
+ * <p>
+ * Get a report (ApplicationReport) of Applications matching the given
+ * application types and application states in the cluster.
+ * </p>
+ *
+ * <p>
+ * If the user does not have <code>VIEW_APP</code> access for an application
+ * then the corresponding report will be filtered as described in
+ * {@link #getApplicationReport(ApplicationId)}.
+ * </p>
+ *
+ * @param applicationTypes
+ * @param applicationStates
+ * @return a list of reports of applications
+ * @throws YarnException
+ * @throws IOException
+ */
+ public abstract List<ApplicationReport> getApplications(
+ Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates) throws YarnException,
+ IOException;
+
+ /**
+ * <p>
* Get metrics ({@link YarnClusterMetrics}) about the cluster.
* </p>
*
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java Tue Aug 27 17:05:13 2013
@@ -211,15 +211,29 @@ public class YarnClientImpl extends Yarn
@Override
public List<ApplicationReport> getApplications() throws YarnException,
IOException {
- return getApplications(null);
+ return getApplications(null, null);
+ }
+
+ @Override
+ public List<ApplicationReport> getApplications(Set<String> applicationTypes)
+ throws YarnException,
+ IOException {
+ return getApplications(applicationTypes, null);
}
@Override
public List<ApplicationReport> getApplications(
- Set<String> applicationTypes) throws YarnException, IOException {
+ EnumSet<YarnApplicationState> applicationStates)
+ throws YarnException, IOException {
+ return getApplications(null, applicationStates);
+ }
+
+ @Override
+ public List<ApplicationReport> getApplications(Set<String> applicationTypes,
+ EnumSet<YarnApplicationState> applicationStates) throws YarnException,
+ IOException {
GetApplicationsRequest request =
- applicationTypes == null ? GetApplicationsRequest.newInstance()
- : GetApplicationsRequest.newInstance(applicationTypes);
+ GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
GetApplicationsResponse response = rmClient.getApplications(request);
return response.getApplicationList();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Tue Aug 27 17:05:13 2013
@@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.text.DecimalFormat;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -49,6 +50,10 @@ public class ApplicationCLI extends Yarn
System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
+ private static final String APP_STATE_CMD ="appStates";
+ private static final String ALLSTATES_OPTION = "ALL";
+
+ private boolean allAppStates;
public static void main(String[] args) throws Exception {
ApplicationCLI cli = new ApplicationCLI();
@@ -66,7 +71,8 @@ public class ApplicationCLI extends Yarn
opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
opts.addOption(LIST_CMD, false, "List applications from the RM. " +
"Supports optional use of --appTypes to filter applications " +
- "based on application type.");
+ "based on application type, " +
+ "and --appStates to filter applications based on application state");
opts.addOption(KILL_CMD, true, "Kills the application.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
Option appTypeOpt = new Option(APP_TYPE_CMD, true,
@@ -75,6 +81,16 @@ public class ApplicationCLI extends Yarn
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
appTypeOpt.setArgName("Comma-separated list of application types");
opts.addOption(appTypeOpt);
+ Option appStateOpt =
+ new Option(
+ APP_STATE_CMD,
+ true,
+ "Works with --list to filter applications based on their state. "
+ + getAllValidApplicationStates());
+ appStateOpt.setValueSeparator(',');
+ appStateOpt.setArgs(Option.UNLIMITED_VALUES);
+ appStateOpt.setArgName("Comma-separated list of application states");
+ opts.addOption(appStateOpt);
opts.getOption(KILL_CMD).setArgName("Application ID");
opts.getOption(STATUS_CMD).setArgName("Application ID");
CommandLine cliParser = new GnuParser().parse(opts, args);
@@ -87,18 +103,44 @@ public class ApplicationCLI extends Yarn
}
printApplicationReport(cliParser.getOptionValue(STATUS_CMD));
} else if (cliParser.hasOption(LIST_CMD)) {
+ allAppStates = false;
Set<String> appTypes = new HashSet<String>();
if(cliParser.hasOption(APP_TYPE_CMD)) {
String[] types = cliParser.getOptionValues(APP_TYPE_CMD);
if (types != null) {
for (String type : types) {
if (!type.trim().isEmpty()) {
- appTypes.add(type.trim());
+ appTypes.add(type.toUpperCase().trim());
}
}
}
}
- listApplications(appTypes);
+
+ EnumSet<YarnApplicationState> appStates =
+ EnumSet.noneOf(YarnApplicationState.class);
+ if (cliParser.hasOption(APP_STATE_CMD)) {
+ String[] states = cliParser.getOptionValues(APP_STATE_CMD);
+ if (states != null) {
+ for (String state : states) {
+ if (!state.trim().isEmpty()) {
+ if (state.trim().equalsIgnoreCase(ALLSTATES_OPTION)) {
+ allAppStates = true;
+ break;
+ }
+ try {
+ appStates.add(YarnApplicationState.valueOf(state.toUpperCase()
+ .trim()));
+ } catch (IllegalArgumentException ex) {
+ sysout.println("The application state " + state
+ + " is invalid.");
+ sysout.println(getAllValidApplicationStates());
+ return exitCode;
+ }
+ }
+ }
+ }
+ }
+ listApplications(appTypes, appStates);
} else if (cliParser.hasOption(KILL_CMD)) {
if (args.length != 2) {
printUsage(opts);
@@ -127,19 +169,33 @@ public class ApplicationCLI extends Yarn
/**
* Lists the applications matching the given application Types
- * present in the Resource Manager
+ * And application States present in the Resource Manager
*
* @param appTypes
+ * @param appStates
* @throws YarnException
* @throws IOException
*/
- private void listApplications(Set<String> appTypes)
- throws YarnException, IOException {
+ private void listApplications(Set<String> appTypes,
+ EnumSet<YarnApplicationState> appStates) throws YarnException,
+ IOException {
PrintWriter writer = new PrintWriter(sysout);
+ if (allAppStates) {
+ for(YarnApplicationState appState : YarnApplicationState.values()) {
+ appStates.add(appState);
+ }
+ } else {
+ if (appStates.isEmpty()) {
+ appStates.add(YarnApplicationState.RUNNING);
+ }
+ }
+
List<ApplicationReport> appsReport =
- client.getApplications(appTypes);
+ client.getApplications(appTypes, appStates);
- writer.println("Total Applications:" + appsReport.size());
+ writer
+ .println("Total number of applications (application-types: " + appTypes
+ + " and states: " + appStates + ")" + ":" + appsReport.size());
writer.printf(APPLICATIONS_PATTERN, "Application-Id",
"Application-Name","Application-Type", "User", "Queue",
"State", "Final-State","Progress", "Tracking-URL");
@@ -229,4 +285,16 @@ public class ApplicationCLI extends Yarn
sysout.println(baos.toString("UTF-8"));
}
+ private String getAllValidApplicationStates() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("The valid application state can be"
+ + " one of the following: ");
+ sb.append(ALLSTATES_OPTION + ",");
+ for (YarnApplicationState appState : YarnApplicationState
+ .values()) {
+ sb.append(appState+",");
+ }
+ String output = sb.toString();
+ return output.substring(0, output.length()-1);
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java Tue Aug 27 17:05:13 2013
@@ -21,11 +21,14 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -40,9 +43,12 @@ import org.apache.hadoop.yarn.util.Conve
@Private
@Unstable
public class NodeCLI extends YarnCLI {
- private static final String NODES_PATTERN = "%16s\t%10s\t%17s\t%18s" +
+ private static final String NODES_PATTERN = "%16s\t%15s\t%17s\t%18s" +
System.getProperty("line.separator");
+ private static final String NODE_STATE_CMD = "states";
+ private static final String NODE_ALL = "all";
+
public static void main(String[] args) throws Exception {
NodeCLI cli = new NodeCLI();
cli.setSysOutPrintStream(System.out);
@@ -57,7 +63,18 @@ public class NodeCLI extends YarnCLI {
Options opts = new Options();
opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
- opts.addOption(LIST_CMD, false, "Lists all the nodes in the RUNNING state.");
+ opts.addOption(LIST_CMD, false, "List all running nodes. " +
+ "Supports optional use of --states to filter nodes " +
+ "based on node state, all --all to list all nodes.");
+ Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
+ "Works with -list to filter nodes based on their states.");
+ nodeStateOpt.setValueSeparator(',');
+ nodeStateOpt.setArgs(Option.UNLIMITED_VALUES);
+ nodeStateOpt.setArgName("Comma-separated list of node states");
+ opts.addOption(nodeStateOpt);
+ Option allOpt = new Option(NODE_ALL, false,
+ "Works with -list to list all nodes.");
+ opts.addOption(allOpt);
CommandLine cliParser = new GnuParser().parse(opts, args);
int exitCode = -1;
@@ -68,7 +85,24 @@ public class NodeCLI extends YarnCLI {
}
printNodeStatus(cliParser.getOptionValue("status"));
} else if (cliParser.hasOption("list")) {
- listClusterNodes();
+ Set<NodeState> nodeStates = new HashSet<NodeState>();
+ if (cliParser.hasOption(NODE_ALL)) {
+ for (NodeState state : NodeState.values()) {
+ nodeStates.add(state);
+ }
+ } else if (cliParser.hasOption(NODE_STATE_CMD)) {
+ String[] types = cliParser.getOptionValues(NODE_STATE_CMD);
+ if (types != null) {
+ for (String type : types) {
+ if (!type.trim().isEmpty()) {
+ nodeStates.add(NodeState.valueOf(type.trim().toUpperCase()));
+ }
+ }
+ }
+ } else {
+ nodeStates.add(NodeState.RUNNING);
+ }
+ listClusterNodes(nodeStates);
} else {
syserr.println("Invalid Command Usage : ");
printUsage(opts);
@@ -86,14 +120,17 @@ public class NodeCLI extends YarnCLI {
}
/**
- * Lists all the nodes present in the cluster
+ * Lists the nodes matching the given node states
*
+ * @param nodeStates
* @throws YarnException
* @throws IOException
*/
- private void listClusterNodes() throws YarnException, IOException {
+ private void listClusterNodes(Set<NodeState> nodeStates)
+ throws YarnException, IOException {
PrintWriter writer = new PrintWriter(sysout);
- List<NodeReport> nodesReport = client.getNodeReports(NodeState.RUNNING);
+ List<NodeReport> nodesReport = client.getNodeReports(
+ nodeStates.toArray(new NodeState[0]));
writer.println("Total Nodes:" + nodesReport.size());
writer.printf(NODES_PATTERN, "Node-Id", "Node-State", "Node-Http-Address",
"Running-Containers");
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Aug 27 17:05:13 2013
@@ -27,6 +27,7 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -55,10 +56,12 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
@@ -163,11 +166,15 @@ public class TestYarnClient {
List<ApplicationReport> expectedReports = ((MockYarnClient)client).getReports();
+ List<ApplicationReport> reports = client.getApplications();
+ Assert.assertEquals(reports, expectedReports);
+
Set<String> appTypes = new HashSet<String>();
appTypes.add("YARN");
appTypes.add("NON-YARN");
- List<ApplicationReport> reports = client.getApplications(appTypes);
+ reports =
+ client.getApplications(appTypes, null);
Assert.assertEquals(reports.size(), 2);
Assert
.assertTrue((reports.get(0).getApplicationType().equals("YARN") && reports
@@ -178,8 +185,28 @@ public class TestYarnClient {
Assert.assertTrue(expectedReports.contains(report));
}
- reports = client.getApplications();
- Assert.assertEquals(reports, expectedReports);
+ EnumSet<YarnApplicationState> appStates =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appStates.add(YarnApplicationState.FINISHED);
+ appStates.add(YarnApplicationState.FAILED);
+ reports = client.getApplications(null, appStates);
+ Assert.assertEquals(reports.size(), 2);
+ Assert
+ .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN") && reports
+ .get(1).getApplicationType().equals("NON-MAPREDUCE"))
+ || (reports.get(1).getApplicationType().equals("NON-YARN") && reports
+ .get(0).getApplicationType().equals("NON-MAPREDUCE")));
+ for (ApplicationReport report : reports) {
+ Assert.assertTrue(expectedReports.contains(report));
+ }
+
+ reports = client.getApplications(appTypes, appStates);
+ Assert.assertEquals(reports.size(), 1);
+ Assert
+ .assertTrue((reports.get(0).getApplicationType().equals("NON-YARN")));
+ for (ApplicationReport report : reports) {
+ Assert.assertTrue(expectedReports.contains(report));
+ }
client.stop();
}
@@ -187,6 +214,8 @@ public class TestYarnClient {
private static class MockYarnClient extends YarnClientImpl {
private ApplicationReport mockReport;
private List<ApplicationReport> reports;
+ GetApplicationsResponse mockAppResponse =
+ mock(GetApplicationsResponse.class);
public MockYarnClient() {
super();
@@ -202,6 +231,8 @@ public class TestYarnClient {
try{
when(rmClient.getApplicationReport(any(
GetApplicationReportRequest.class))).thenReturn(mockResponse);
+ when(rmClient.getApplications(any(GetApplicationsRequest.class)))
+ .thenReturn(mockAppResponse);
} catch (YarnException e) {
Assert.fail("Exception is not expected.");
} catch (IOException e) {
@@ -212,16 +243,11 @@ public class TestYarnClient {
@Override
public List<ApplicationReport> getApplications(
- Set<String> applicationTypes) throws YarnException, IOException {
- GetApplicationsRequest request =
- applicationTypes == null ? GetApplicationsRequest.newInstance()
- : GetApplicationsRequest.newInstance(applicationTypes);
- when(rmClient.getApplications(request))
- .thenReturn(
- getApplicationReports(reports,
- request));
- GetApplicationsResponse response = rmClient.getApplications(request);
- return response.getApplicationList();
+ Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates)
+ throws YarnException, IOException {
+ when(mockAppResponse.getApplicationList()).thenReturn(
+ getApplicationReports(reports, applicationTypes, applicationStates));
+ return super.getApplications(applicationTypes, applicationStates);
}
@Override
@@ -243,7 +269,7 @@ public class TestYarnClient {
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
- YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
List<ApplicationReport> applicationReports =
new ArrayList<ApplicationReport>();
@@ -262,31 +288,44 @@ public class TestYarnClient {
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null,
- YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
+ YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
null);
applicationReports.add(newApplicationReport3);
+
+ ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
+ ApplicationReport newApplicationReport4 =
+ ApplicationReport.newInstance(
+ applicationId4,
+ ApplicationAttemptId.newInstance(applicationId4, 4),
+ "user4", "queue4", "appname4", "host4", 127, null,
+ YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
+ "NON-MAPREDUCE", null);
+ applicationReports.add(newApplicationReport4);
return applicationReports;
}
- private GetApplicationsResponse getApplicationReports(
+ private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports,
- GetApplicationsRequest request) {
+ Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
- Set<String> appTypes = request.getApplicationTypes();
- boolean bypassFilter = appTypes.isEmpty();
-
for (ApplicationReport appReport : applicationReports) {
- if (!(bypassFilter || appTypes.contains(
- appReport.getApplicationType()))) {
- continue;
+ if (applicationTypes != null && !applicationTypes.isEmpty()) {
+ if (!applicationTypes.contains(appReport.getApplicationType())) {
+ continue;
+ }
+ }
+
+ if (applicationStates != null && !applicationStates.isEmpty()) {
+ if (!applicationStates.contains(appReport.getYarnApplicationState())) {
+ continue;
+ }
}
appReports.add(appReport);
}
- GetApplicationsResponse response =
- GetApplicationsResponse.newInstance(appReports);
- return response;
+ return appReports;
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Tue Aug 27 17:05:13 2013
@@ -33,6 +33,7 @@ import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Date;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -134,7 +135,7 @@ public class TestYarnCLI {
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
- YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
@@ -152,23 +153,39 @@ public class TestYarnCLI {
ApplicationReport newApplicationReport3 = ApplicationReport.newInstance(
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null,
- YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
+ YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
null);
applicationReports.add(newApplicationReport3);
- Set<String> appType1 = new HashSet<String>();
- appType1.add("YARN");
+ ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
+ ApplicationReport newApplicationReport4 = ApplicationReport.newInstance(
+ applicationId4, ApplicationAttemptId.newInstance(applicationId4, 4),
+ "user4", "queue4", "appname4", "host4", 127, null,
+ YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
+ null);
+ applicationReports.add(newApplicationReport4);
- when(client.getApplications(appType1)).thenReturn(
- getApplicationReports(applicationReports, appType1));
- int result = cli.run(new String[] { "-list", "-appTypes", "YARN" });
+ // Test command yarn application -list
+ // if the set appStates is empty, RUNNING state will be automatically added
+ // to the appStates list
+ // the output of yarn application -list should be the same as
+ // equals to yarn application -list --appStates RUNNING
+ Set<String> appType1 = new HashSet<String>();
+ EnumSet<YarnApplicationState> appState1 =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appState1.add(YarnApplicationState.RUNNING);
+ when(client.getApplications(appType1, appState1)).thenReturn(
+ getApplicationReports(applicationReports, appType1, appState1, false));
+ int result = cli.run(new String[] { "-list" });
assertEquals(0, result);
- verify(client).getApplications(appType1);
+ verify(client).getApplications(appType1, appState1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
- pw.println("Total Applications:1");
+ pw.println("Total number of applications (application-types: " + appType1
+ + " and states: " + appState1 + ")" + ":" + 2);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -176,27 +193,41 @@ public class TestYarnCLI {
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t ");
- pw.print("queue\t FINISHED\t ");
+ pw.print("queue\t RUNNING\t ");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
+ pw.print(" application_1234_0007\t ");
+ pw.print("appname3\t MAPREDUCE\t user3\t ");
+ pw.print("queue3\t RUNNING\t ");
+ pw.print("SUCCEEDED\t 73.79%");
+ pw.println("\t N/A");
pw.close();
String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
+ //Test command yarn application -list --appTypes apptype1,apptype2
+ //the output should be the same as
+ //yarn application -list --appTypes apptyp1, apptype2 --appStates RUNNING
sysOutStream.reset();
Set<String> appType2 = new HashSet<String>();
appType2.add("YARN");
- appType2.add("FOO-YARN");
- when(client.getApplications(appType2)).thenReturn(
- getApplicationReports(applicationReports, appType2));
- cli.run(new String[] { "-list", "-appTypes", "YARN , ,, ,FOO-YARN",
- ",,,,, YARN,," });
+ appType2.add("NON-YARN");
+
+ EnumSet<YarnApplicationState> appState2 =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appState2.add(YarnApplicationState.RUNNING);
+ when(client.getApplications(appType2, appState2)).thenReturn(
+ getApplicationReports(applicationReports, appType2, appState2, false));
+ result =
+ cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN",
+ " ,, ,," });
assertEquals(0, result);
- verify(client).getApplications(appType2);
+ verify(client).getApplications(appType2, appState2);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
- pw.println("Total Applications:1");
+ pw.println("Total number of applications (application-types: " + appType2
+ + " and states: " + appState2 + ")" + ":" + 1);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -204,7 +235,7 @@ public class TestYarnCLI {
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t ");
- pw.print("queue\t FINISHED\t ");
+ pw.print("queue\t RUNNING\t ");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
pw.close();
@@ -212,50 +243,122 @@ public class TestYarnCLI {
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
+ //Test command yarn application -list --appStates appState1,appState2
sysOutStream.reset();
Set<String> appType3 = new HashSet<String>();
- appType3.add("YARN");
- appType3.add("NON-YARN");
- when(client.getApplications(appType3)).thenReturn(
- getApplicationReports(applicationReports, appType3));
- result = cli.run(new String[] { "-list", "-appTypes", "YARN,NON-YARN" });
+ EnumSet<YarnApplicationState> appState3 =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appState3.add(YarnApplicationState.FINISHED);
+ appState3.add(YarnApplicationState.FAILED);
+
+ when(client.getApplications(appType3, appState3)).thenReturn(
+ getApplicationReports(applicationReports, appType3, appState3, false));
+ result =
+ cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , FAILED",
+ ",,FINISHED" });
assertEquals(0, result);
- verify(client).getApplications(appType3);
+ verify(client).getApplications(appType3, appState3);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
- pw.println("Total Applications:2");
+ pw.println("Total number of applications (application-types: " + appType3
+ + " and states: " + appState3 + ")" + ":" + 2);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
pw.print("Final-State\t Progress");
pw.println("\t Tracking-URL");
- pw.print(" application_1234_0005\t ");
- pw.print("appname\t YARN\t user\t ");
- pw.print("queue\t FINISHED\t ");
- pw.print("SUCCEEDED\t 53.79%");
- pw.println("\t N/A");
pw.print(" application_1234_0006\t ");
pw.print("appname2\t NON-YARN\t user2\t ");
pw.print("queue2\t FINISHED\t ");
pw.print("SUCCEEDED\t 63.79%");
pw.println("\t N/A");
+ pw.print(" application_1234_0008\t ");
+ pw.print("appname4\t NON-MAPREDUCE\t user4\t ");
+ pw.print("queue4\t FAILED\t ");
+ pw.print("SUCCEEDED\t 83.79%");
+ pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
+ // Test command yarn application -list --appTypes apptype1,apptype2
+ // --appStates appstate1,appstate2
sysOutStream.reset();
Set<String> appType4 = new HashSet<String>();
- when(client.getApplications(appType4)).thenReturn(
- getApplicationReports(applicationReports, appType4));
- result = cli.run(new String[] { "-list" });
+ appType4.add("YARN");
+ appType4.add("NON-YARN");
+
+ EnumSet<YarnApplicationState> appState4 =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appState4.add(YarnApplicationState.FINISHED);
+ appState4.add(YarnApplicationState.FAILED);
+
+ when(client.getApplications(appType4, appState4)).thenReturn(
+ getApplicationReports(applicationReports, appType4, appState4, false));
+ result =
+ cli.run(new String[] { "-list", "--appTypes", "YARN,NON-YARN",
+ "--appStates", "FINISHED ,, , FAILED" });
assertEquals(0, result);
- verify(client).getApplications(appType4);
+ verify(client).getApplications(appType2, appState2);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType4
+ + " and states: " + appState4 + ")" + ":" + 1);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0006\t ");
+ pw.print("appname2\t NON-YARN\t user2\t ");
+ pw.print("queue2\t FINISHED\t ");
+ pw.print("SUCCEEDED\t 63.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+ //Test command yarn application -list --appStates with invalid appStates
+ sysOutStream.reset();
+ result =
+ cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , INVALID" });
+ assertEquals(-1, result);
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
- pw.println("Total Applications:3");
+ pw.println("The application state INVALID is invalid.");
+ pw.print("The valid application state can be one of the following: ");
+ StringBuilder sb = new StringBuilder();
+ sb.append("ALL,");
+ for(YarnApplicationState state : YarnApplicationState.values()) {
+ sb.append(state+",");
+ }
+ String output = sb.toString();
+ pw.println(output.substring(0, output.length()-1));
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+
+ //Test command yarn application -list --appStates all
+ sysOutStream.reset();
+ Set<String> appType5 = new HashSet<String>();
+
+ EnumSet<YarnApplicationState> appState5 =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appState5.add(YarnApplicationState.FINISHED);
+ when(client.getApplications(appType5, appState5)).thenReturn(
+ getApplicationReports(applicationReports, appType5, appState5, true));
+ result =
+ cli.run(new String[] { "-list", "--appStates", "FINISHED ,, , ALL" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType5, appState5);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType5
+ + " and states: " + appState5 + ")" + ":" + 4);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -263,7 +366,7 @@ public class TestYarnCLI {
pw.println("\t Tracking-URL");
pw.print(" application_1234_0005\t ");
pw.print("appname\t YARN\t user\t ");
- pw.print("queue\t FINISHED\t ");
+ pw.print("queue\t RUNNING\t ");
pw.print("SUCCEEDED\t 53.79%");
pw.println("\t N/A");
pw.print(" application_1234_0006\t ");
@@ -273,27 +376,80 @@ public class TestYarnCLI {
pw.println("\t N/A");
pw.print(" application_1234_0007\t ");
pw.print("appname3\t MAPREDUCE\t user3\t ");
- pw.print("queue3\t FINISHED\t ");
+ pw.print("queue3\t RUNNING\t ");
pw.print("SUCCEEDED\t 73.79%");
pw.println("\t N/A");
+ pw.print(" application_1234_0008\t ");
+ pw.print("appname4\t NON-MAPREDUCE\t user4\t ");
+ pw.print("queue4\t FAILED\t ");
+ pw.print("SUCCEEDED\t 83.79%");
+ pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
- verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+ verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
+
+ // Test command yarn application user case insensitive
+ sysOutStream.reset();
+ Set<String> appType6 = new HashSet<String>();
+ appType6.add("YARN");
+ appType6.add("NON-YARN");
+
+ EnumSet<YarnApplicationState> appState6 =
+ EnumSet.noneOf(YarnApplicationState.class);
+ appState6.add(YarnApplicationState.FINISHED);
+ when(client.getApplications(appType6, appState6)).thenReturn(
+ getApplicationReports(applicationReports, appType6, appState6, false));
+ result =
+ cli.run(new String[] { "-list", "-appTypes", "YARN, ,, NON-YARN",
+ "--appStates", "finished" });
+ assertEquals(0, result);
+ verify(client).getApplications(appType6, appState6);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total number of applications (application-types: " + appType6
+ + " and states: " + appState6 + ")" + ":" + 1);
+ pw.print(" Application-Id\t Application-Name");
+ pw.print("\t Application-Type");
+ pw.print("\t User\t Queue\t State\t ");
+ pw.print("Final-State\t Progress");
+ pw.println("\t Tracking-URL");
+ pw.print(" application_1234_0006\t ");
+ pw.print("appname2\t NON-YARN\t user2\t ");
+ pw.print("queue2\t FINISHED\t ");
+ pw.print("SUCCEEDED\t 63.79%");
+ pw.println("\t N/A");
+ pw.close();
+ appsReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appsReportStr, sysOutStream.toString());
+ verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
}
private List<ApplicationReport> getApplicationReports(
List<ApplicationReport> applicationReports,
- Set<String> appTypes) {
+ Set<String> appTypes, EnumSet<YarnApplicationState> appStates,
+ boolean allStates) {
List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
- boolean bypassFilter = appTypes.isEmpty();
+ if (allStates) {
+ for(YarnApplicationState state : YarnApplicationState.values()) {
+ appStates.add(state);
+ }
+ }
for (ApplicationReport appReport : applicationReports) {
- if (!(bypassFilter || appTypes.contains(
- appReport.getApplicationType()))) {
- continue;
+ if (appTypes != null && !appTypes.isEmpty()) {
+ if (!appTypes.contains(appReport.getApplicationType())) {
+ continue;
+ }
}
+
+ if (appStates != null && !appStates.isEmpty()) {
+ if (!appStates.contains(appReport.getYarnApplicationState())) {
+ continue;
+ }
+ }
+
appReports.add(appReport);
}
return appReports;
@@ -363,36 +519,239 @@ public class TestYarnCLI {
@Test
public void testListClusterNodes() throws Exception {
+ List<NodeReport> nodeReports = new ArrayList<NodeReport>();
+ nodeReports.addAll(getNodeReports(1, NodeState.NEW));
+ nodeReports.addAll(getNodeReports(2, NodeState.RUNNING));
+ nodeReports.addAll(getNodeReports(1, NodeState.UNHEALTHY));
+ nodeReports.addAll(getNodeReports(1, NodeState.DECOMMISSIONED));
+ nodeReports.addAll(getNodeReports(1, NodeState.REBOOTED));
+ nodeReports.addAll(getNodeReports(1, NodeState.LOST));
+
NodeCLI cli = new NodeCLI();
- when(client.getNodeReports(NodeState.RUNNING)).thenReturn(
- getNodeReports(3));
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
- int result = cli.run(new String[] { "-list" });
+
+ Set<NodeState> nodeStates = new HashSet<NodeState>();
+ nodeStates.add(NodeState.NEW);
+ NodeState[] states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ int result = cli.run(new String[] { "-list", "--states", "NEW" });
assertEquals(0, result);
- verify(client).getNodeReports(NodeState.RUNNING);
+ verify(client).getNodeReports(states);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
- pw.println("Total Nodes:3");
- pw.print(" Node-Id\tNode-State\tNode-Http-Address\t");
+ pw.println("Total Nodes:1");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
pw.println("Running-Containers");
- pw.print(" host0:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host1:0\t RUNNING\t host1:8888");
- pw.println("\t 0");
- pw.print(" host2:0\t RUNNING\t host1:8888");
+ pw.print(" host0:0\t NEW\t host1:8888");
pw.println("\t 0");
pw.close();
String nodesReportStr = baos.toString("UTF-8");
Assert.assertEquals(nodesReportStr, sysOutStream.toString());
verify(sysOut, times(1)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ nodeStates.add(NodeState.RUNNING);
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--states", "RUNNING" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:2");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t RUNNING\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host1:0\t RUNNING\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(2)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ result = cli.run(new String[] { "-list" });
+ assertEquals(0, result);
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(3)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ nodeStates.add(NodeState.UNHEALTHY);
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--states", "UNHEALTHY" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:1");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t UNHEALTHY\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(4)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ nodeStates.add(NodeState.DECOMMISSIONED);
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--states", "DECOMMISSIONED" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:1");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t DECOMMISSIONED\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(5)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ nodeStates.add(NodeState.REBOOTED);
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--states", "REBOOTED" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:1");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t REBOOTED\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(6)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ nodeStates.add(NodeState.LOST);
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--states", "LOST" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:1");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t LOST\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(7)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ nodeStates.add(NodeState.NEW);
+ nodeStates.add(NodeState.RUNNING);
+ nodeStates.add(NodeState.LOST);
+ nodeStates.add(NodeState.REBOOTED);
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--states",
+ "NEW,RUNNING,LOST,REBOOTED" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:5");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t NEW\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t RUNNING\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host1:0\t RUNNING\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t REBOOTED\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t LOST\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(8)).write(any(byte[].class), anyInt(), anyInt());
+
+ sysOutStream.reset();
+ nodeStates.clear();
+ for (NodeState s : NodeState.values()) {
+ nodeStates.add(s);
+ }
+ states = nodeStates.toArray(new NodeState[0]);
+ when(client.getNodeReports(states))
+ .thenReturn(getNodeReports(nodeReports, nodeStates));
+ result = cli.run(new String[] { "-list", "--all" });
+ assertEquals(0, result);
+ verify(client).getNodeReports(states);
+ baos = new ByteArrayOutputStream();
+ pw = new PrintWriter(baos);
+ pw.println("Total Nodes:7");
+ pw.print(" Node-Id\t Node-State\tNode-Http-Address\t");
+ pw.println("Running-Containers");
+ pw.print(" host0:0\t NEW\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t RUNNING\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host1:0\t RUNNING\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t UNHEALTHY\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t DECOMMISSIONED\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t REBOOTED\t host1:8888");
+ pw.println("\t 0");
+ pw.print(" host0:0\t LOST\t host1:8888");
+ pw.println("\t 0");
+ pw.close();
+ nodesReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(nodesReportStr, sysOutStream.toString());
+ verify(sysOut, times(9)).write(any(byte[].class), anyInt(), anyInt());
+ }
+
+ private List<NodeReport> getNodeReports(
+ List<NodeReport> nodeReports,
+ Set<NodeState> nodeStates) {
+ List<NodeReport> reports = new ArrayList<NodeReport>();
+
+ for (NodeReport nodeReport : nodeReports) {
+ if (nodeStates.contains(nodeReport.getNodeState())) {
+ reports.add(nodeReport);
+ }
+ }
+ return reports;
}
@Test
public void testNodeStatus() throws Exception {
NodeId nodeId = NodeId.newInstance("host0", 0);
NodeCLI cli = new NodeCLI();
- when(client.getNodeReports()).thenReturn(getNodeReports(3));
+ when(client.getNodeReports()).thenReturn(
+ getNodeReports(3, NodeState.RUNNING));
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
@@ -424,7 +783,8 @@ public class TestYarnCLI {
public void testAbsentNodeStatus() throws Exception {
NodeId nodeId = NodeId.newInstance("Absenthost0", 0);
NodeCLI cli = new NodeCLI();
- when(client.getNodeReports()).thenReturn(getNodeReports(0));
+ when(client.getNodeReports()).thenReturn(
+ getNodeReports(0, NodeState.RUNNING));
cli.setClient(client);
cli.setSysOutPrintStream(sysOut);
cli.setSysErrPrintStream(sysErr);
@@ -452,12 +812,12 @@ public class TestYarnCLI {
verify(sysErr).println("Invalid Command Usage : ");
}
- private List<NodeReport> getNodeReports(int noOfNodes) {
+ private List<NodeReport> getNodeReports(int noOfNodes, NodeState state) {
List<NodeReport> nodeReports = new ArrayList<NodeReport>();
for (int i = 0; i < noOfNodes; i++) {
NodeReport nodeReport = NodeReport.newInstance(NodeId
- .newInstance("host" + i, 0), NodeState.RUNNING, "host" + 1 + ":8888",
+ .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
"rack1", Records.newRecord(Resource.class), Records
.newRecord(Resource.class), 0, "", 0);
nodeReports.add(nodeReport);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Tue Aug 27 17:05:13 2013
@@ -18,13 +18,18 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+import java.util.EnumSet;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
@@ -38,6 +43,7 @@ public class GetApplicationsRequestPBImp
boolean viaProto = false;
Set<String> applicationTypes = null;
+ EnumSet<YarnApplicationState> applicationStates = null;
public GetApplicationsRequestPBImpl() {
builder = GetApplicationsRequestProto.newBuilder();
@@ -67,6 +73,40 @@ public class GetApplicationsRequestPBImp
if (this.applicationTypes != null) {
addLocalApplicationTypesToProto();
}
+ if (this.applicationStates != null) {
+ maybeInitBuilder();
+ builder.clearApplicationStates();
+ Iterable<YarnApplicationStateProto> iterable =
+ new Iterable<YarnApplicationStateProto>() {
+
+ @Override
+ public Iterator<YarnApplicationStateProto> iterator() {
+ return new Iterator<YarnApplicationStateProto>() {
+
+ Iterator<YarnApplicationState> iter = applicationStates
+ .iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public YarnApplicationStateProto next() {
+ return ProtoUtils.convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ builder.addAllApplicationStates(iterable);
+ }
}
private void addLocalApplicationTypesToProto() {
@@ -94,6 +134,20 @@ public class GetApplicationsRequestPBImp
this.applicationTypes.addAll(appTypeList);
}
+ private void initApplicationStates() {
+ if (this.applicationStates != null) {
+ return;
+ }
+ GetApplicationsRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List<YarnApplicationStateProto> appStatesList =
+ p.getApplicationStatesList();
+ this.applicationStates = EnumSet.noneOf(YarnApplicationState.class);
+
+ for (YarnApplicationStateProto c : appStatesList) {
+ this.applicationStates.add(ProtoUtils.convertFromProtoFormat(c));
+ }
+ }
+
@Override
public Set<String> getApplicationTypes() {
initApplicationTypes();
@@ -109,6 +163,21 @@ public class GetApplicationsRequestPBImp
}
@Override
+ public EnumSet<YarnApplicationState> getApplicationStates() {
+ initApplicationStates();
+ return this.applicationStates;
+ }
+
+ @Override
+ public void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
+ maybeInitBuilder();
+ if (applicationStates == null) {
+ builder.clearApplicationStates();
+ }
+ this.applicationStates = applicationStates;
+ }
+
+ @Override
public int hashCode() {
return getProto().hashCode();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Tue Aug 27 17:05:13 2013
@@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUti
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,7 +67,6 @@ import com.google.inject.servlet.GuiceFi
@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
public class WebApps {
static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
-
public static class Builder<T> {
static class ServletStruct {
public Class<? extends HttpServlet> clazz;
@@ -82,6 +83,8 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
boolean devMode = false;
+ private String spnegoPrincipalKey;
+ private String spnegoKeytabKey;
private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
private final HashMap<String, Object> attributes = new HashMap<String, Object>();
@@ -135,6 +138,16 @@ public class WebApps {
this.conf = conf;
return this;
}
+
+ public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
+ this.spnegoPrincipalKey = spnegoPrincipalKey;
+ return this;
+ }
+
+ public Builder<T> withHttpSpnegoKeytabKey(String spnegoKeytabKey) {
+ this.spnegoKeytabKey = spnegoKeytabKey;
+ return this;
+ }
public Builder<T> inDevMode() {
devMode = true;
@@ -197,8 +210,32 @@ public class WebApps {
}
}
HttpServer server =
- new HttpServer(name, bindAddress, port, findPort, conf,
- new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs());
+ new HttpServer(name, bindAddress, port, findPort, conf,
+ new AdminACLsManager(conf).getAdminAcl(), null,
+ webapp.getServePathSpecs()) {
+
+ {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ boolean initSpnego = true;
+ if (spnegoPrincipalKey == null
+ || conf.get(spnegoPrincipalKey, "").isEmpty()) {
+ LOG.warn("Principal for spnego filter is not set");
+ initSpnego = false;
+ }
+ if (spnegoKeytabKey == null
+ || conf.get(spnegoKeytabKey, "").isEmpty()) {
+ LOG.warn("Keytab for spnego filter is not set");
+ initSpnego = false;
+ }
+ if (initSpnego) {
+ LOG.info("Initializing spnego filter with principal key : "
+ + spnegoPrincipalKey + " keytab key : "
+ + spnegoKeytabKey);
+ initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey);
+ }
+ }
+ }
+ };
for(ServletStruct struct: servlets) {
server.addServlet(struct.name, struct.spec, struct.clazz);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1517887&r1=1517886&r2=1517887&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Tue Aug 27 17:05:13 2013
@@ -59,8 +59,15 @@ public class WebServer extends AbstractS
LOG.info("Instantiating NMWebApp at " + bindAddress);
try {
this.webApp =
- WebApps.$for("node", Context.class, this.nmContext, "ws")
- .at(bindAddress).with(getConfig()).start(this.nmWebApp);
+ WebApps
+ .$for("node", Context.class, this.nmContext, "ws")
+ .at(bindAddress)
+ .with(getConfig())
+ .withHttpSpnegoPrincipalKey(
+ YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .withHttpSpnegoKeytabKey(
+ YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .start(this.nmWebApp);
this.port = this.webApp.httpServer().getPort();
} catch (Exception e) {
String msg = "NMWebapps failed to start.";