You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/09/04 01:20:48 UTC

svn commit: r1519885 [1/3] - in /hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/ hadoop-yarn/...

Author: arp
Date: Tue Sep  3 23:20:45 2013
New Revision: 1519885

URL: http://svn.apache.org/r1519885
Log:
Merging r1518852 through r1519883 from trunk to branch HDFS-2832

Added:
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
      - copied unchanged from r1519883, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
      - copied unchanged from r1519883, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
      - copied unchanged from r1519883, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
Modified:
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
    hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Tue Sep  3 23:20:45 2013
@@ -21,6 +21,9 @@ Release 2.3.0 - UNRELEASED
 
   NEW FEATURES
 
+    YARN-649. Added a new NM web-service to serve container logs in plain text
+    over HTTP. (Sandy Ryza via vinodkv)
+
   IMPROVEMENTS
 
     YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
@@ -39,6 +42,9 @@ Release 2.1.1-beta - UNRELEASED
 
   INCOMPATIBLE CHANGES
 
+    YARN-707. Added user information also in the YARN ClientToken so that AMs
+    can implement authorization based on incoming users. (Jason Lowe via vinodkv)
+
   NEW FEATURES
 
   IMPROVEMENTS
@@ -66,6 +72,20 @@ Release 2.1.1-beta - UNRELEASED
     YARN-1080. Improved help message for "yarn logs" command. (Xuan Gong via
     vinodkv)
 
+    YARN-771. AMRMClient support for resource blacklisting (Junping Du via
+    bikas)
+
+    YARN-1117. Improved help messages for "yarn application" and "yarn node"
+    commands. (Xuan Gong via vinodkv)
+
+    YARN-1120. Made ApplicationConstants.Environment.USER definition OS neutral
+    as the corresponding value is now set correctly end-to-end. (Chuan Liu via
+    vinodkv)
+
+    YARN-1124. Modified YARN CLI application list to display new and submitted
+    applications together with running apps by default, following up YARN-1074.
+    (Xuan Gong via vinodkv)
+
   OPTIMIZATIONS
 
   BUG FIXES
@@ -126,6 +146,9 @@ Release 2.1.1-beta - UNRELEASED
     YARN-1101. Active nodes can be decremented below 0 (Robert Parker 
     via tgraves)
 
+    YARN-1077. Fixed TestContainerLaunch test failure on Windows. (Chuan Liu via
+    vinodkv)
+
 Release 2.1.0-beta - 2013-08-22
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Tue Sep  3 23:20:45 2013
@@ -80,7 +80,7 @@ public interface ApplicationConstants {
      * $USER
      * Final, non-modifiable.
      */
-    USER(Shell.WINDOWS ? "USERNAME": "USER"),
+    USER("USER"),
     
     /**
      * $LOGNAME

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java Tue Sep  3 23:20:45 2013
@@ -286,4 +286,15 @@ public abstract class AMRMClient<T exten
                                            Priority priority, 
                                            String resourceName, 
                                            Resource capability);
+  
+  /**
+   * Update application's blacklist with addition or removal resources.
+   * 
+   * @param blacklistAdditions list of resources which should be added to the 
+   *        application blacklist
+   * @param blacklistRemovals list of resources which should be removed from the 
+   *        application blacklist
+   */
+  public abstract void updateBlacklist(List<String> blacklistAdditions,
+      List<String> blacklistRemovals);
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Tue Sep  3 23:20:45 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.record
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.Priority;
 import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
 import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.client.ClientRMProxy;
 import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -80,6 +81,9 @@ public class AMRMClientImpl<T extends Co
   protected Resource clusterAvailableResources;
   protected int clusterNodeCount;
   
+  protected final Set<String> blacklistAdditions = new HashSet<String>();
+  protected final Set<String> blacklistRemovals = new HashSet<String>();
+  
   class ResourceRequestInfo {
     ResourceRequest remoteRequest;
     LinkedHashSet<T> containerRequests;
@@ -199,9 +203,11 @@ public class AMRMClientImpl<T extends Co
     Preconditions.checkArgument(progressIndicator >= 0,
         "Progress indicator should not be negative");
     AllocateResponse allocateResponse = null;
-    ArrayList<ResourceRequest> askList = null;
-    ArrayList<ContainerId> releaseList = null;
+    List<ResourceRequest> askList = null;
+    List<ContainerId> releaseList = null;
     AllocateRequest allocateRequest = null;
+    List<String> blacklistToAdd = new ArrayList<String>();
+    List<String> blacklistToRemove = new ArrayList<String>();
     
     try {
       synchronized (this) {
@@ -217,9 +223,22 @@ public class AMRMClientImpl<T extends Co
         // optimistically clear this collection assuming no RPC failure
         ask.clear();
         release.clear();
+
+        blacklistToAdd.addAll(blacklistAdditions);
+        blacklistToRemove.addAll(blacklistRemovals);
+        
+        ResourceBlacklistRequest blacklistRequest = 
+            (blacklistToAdd != null) || (blacklistToRemove != null) ? 
+            ResourceBlacklistRequest.newInstance(blacklistToAdd,
+                blacklistToRemove) : null;
+        
         allocateRequest =
             AllocateRequest.newInstance(lastResponseId, progressIndicator,
-              askList, releaseList, null);
+              askList, releaseList, blacklistRequest);
+        // clear blacklistAdditions and blacklistRemovals before 
+        // unsynchronized part
+        blacklistAdditions.clear();
+        blacklistRemovals.clear();
       }
 
       allocateResponse = rmClient.allocate(allocateRequest);
@@ -253,6 +272,9 @@ public class AMRMClientImpl<T extends Co
               ask.add(oldAsk);
             }
           }
+          
+          blacklistAdditions.addAll(blacklistToAdd);
+          blacklistRemovals.addAll(blacklistToRemove);
         }
       }
     }
@@ -604,4 +626,31 @@ public class AMRMClientImpl<T extends Co
           + " #asks=" + ask.size());
     }
   }
+
+  @Override
+  public synchronized void updateBlacklist(List<String> blacklistAdditions,
+      List<String> blacklistRemovals) {
+    
+    if (blacklistAdditions != null) {
+      this.blacklistAdditions.addAll(blacklistAdditions);
+      // if some resources are also in blacklistRemovals updated before, we 
+      // should remove them here.
+      this.blacklistRemovals.removeAll(blacklistAdditions);
+    }
+    
+    if (blacklistRemovals != null) {
+      this.blacklistRemovals.addAll(blacklistRemovals);
+      // if some resources are in blacklistAdditions before, we should remove
+      // them here.
+      this.blacklistAdditions.removeAll(blacklistRemovals);
+    }
+    
+    if (blacklistAdditions != null && blacklistRemovals != null
+        && blacklistAdditions.removeAll(blacklistRemovals)) {
+      // we allow resources to appear in addition list and removal list in the
+      // same invocation of updateBlacklist(), but should get a warn here.
+      LOG.warn("The same resources appear in both blacklistAdditions and " +
+          "blacklistRemovals in updateBlacklist.");
+    }
+  }
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Tue Sep  3 23:20:45 2013
@@ -29,6 +29,7 @@ import java.util.Set;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -70,32 +71,38 @@ public class ApplicationCLI extends Yarn
     Options opts = new Options();
     opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
     opts.addOption(LIST_CMD, false, "List applications from the RM. " +
-        "Supports optional use of --appTypes to filter applications " +
+        "Supports optional use of -appTypes to filter applications " +
         "based on application type, " +
-        "and --appStates to filter applications based on application state");
+        "and -appStates to filter applications based on application state");
     opts.addOption(KILL_CMD, true, "Kills the application.");
     opts.addOption(HELP_CMD, false, "Displays help for all commands.");
-    Option appTypeOpt = new Option(APP_TYPE_CMD, true,
-        "Works with --list to filter applications based on their type.");
+    Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " +
+        "filter applications based on " +
+        "input comma-separated list of application types.");
     appTypeOpt.setValueSeparator(',');
     appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
-    appTypeOpt.setArgName("Comma-separated list of application types");
+    appTypeOpt.setArgName("Types");
     opts.addOption(appTypeOpt);
-    Option appStateOpt =
-        new Option(
-            APP_STATE_CMD,
-            true,
-            "Works with --list to filter applications based on their state. "
-                + getAllValidApplicationStates());
+    Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " +
+        "to filter applications based on input comma-separated list of " +
+        "application states. " + getAllValidApplicationStates());
     appStateOpt.setValueSeparator(',');
     appStateOpt.setArgs(Option.UNLIMITED_VALUES);
-    appStateOpt.setArgName("Comma-separated list of application states");
+    appStateOpt.setArgName("States");
     opts.addOption(appStateOpt);
     opts.getOption(KILL_CMD).setArgName("Application ID");
     opts.getOption(STATUS_CMD).setArgName("Application ID");
-    CommandLine cliParser = new GnuParser().parse(opts, args);
 
     int exitCode = -1;
+    CommandLine cliParser = null;
+    try {
+      cliParser = new GnuParser().parse(opts, args);
+    } catch (MissingArgumentException ex) {
+      sysout.println("Missing argument for options");
+      printUsage(opts);
+      return exitCode;
+    }
+
     if (cliParser.hasOption(STATUS_CMD)) {
       if (args.length != 2) {
         printUsage(opts);
@@ -187,6 +194,8 @@ public class ApplicationCLI extends Yarn
     } else {
       if (appStates.isEmpty()) {
         appStates.add(YarnApplicationState.RUNNING);
+        appStates.add(YarnApplicationState.ACCEPTED);
+        appStates.add(YarnApplicationState.SUBMITTED);
       }
     }
 

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java Tue Sep  3 23:20:45 2013
@@ -28,6 +28,7 @@ import java.util.Set;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang.time.DateFormatUtils;
@@ -64,20 +65,29 @@ public class NodeCLI extends YarnCLI {
     Options opts = new Options();
     opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
     opts.addOption(LIST_CMD, false, "List all running nodes. " +
-        "Supports optional use of --states to filter nodes " +
-        "based on node state, all --all to list all nodes.");
+        "Supports optional use of -states to filter nodes " +
+        "based on node state, all -all to list all nodes.");
     Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
-        "Works with -list to filter nodes based on their states.");
+        "Works with -list to filter nodes based on input comma-separated list of node states.");
     nodeStateOpt.setValueSeparator(',');
     nodeStateOpt.setArgs(Option.UNLIMITED_VALUES);
-    nodeStateOpt.setArgName("Comma-separated list of node states");
+    nodeStateOpt.setArgName("States");
     opts.addOption(nodeStateOpt);
     Option allOpt = new Option(NODE_ALL, false,
         "Works with -list to list all nodes.");
     opts.addOption(allOpt);
-    CommandLine cliParser = new GnuParser().parse(opts, args);
+    opts.getOption(STATUS_CMD).setArgName("NodeId");
 
     int exitCode = -1;
+    CommandLine cliParser = null;
+    try {
+      cliParser = new GnuParser().parse(opts, args);
+    } catch (MissingArgumentException ex) {
+      sysout.println("Missing argument for options");
+      printUsage(opts);
+      return exitCode;
+    }
+
     if (cliParser.hasOption("status")) {
       if (args.length != 2) {
         printUsage(opts);

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Tue Sep  3 23:20:45 2013
@@ -18,13 +18,16 @@
 
 package org.apache.hadoop.yarn.client.api.impl;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -97,6 +100,7 @@ public class TestAMRMClient {
   static String rack;
   static String[] nodes;
   static String[] racks;
+  private final static int DEFAULT_ITERATION = 3;
   
   @BeforeClass
   public static void setup() throws Exception {
@@ -476,6 +480,144 @@ public class TestAMRMClient {
       }
     }
   }
+  
+  @Test (timeout=60000)
+  public void testAllocationWithBlacklist() throws YarnException, IOException {
+    AMRMClientImpl<ContainerRequest> amClient = null;
+    try {
+      // start am rm client
+      amClient =
+          (AMRMClientImpl<ContainerRequest>) AMRMClient
+            .<ContainerRequest> createAMRMClient();
+      amClient.init(conf);
+      amClient.start();
+      amClient.registerApplicationMaster("Host", 10000, "");
+      
+      assertTrue(amClient.ask.size() == 0);
+      assertTrue(amClient.release.size() == 0);
+      
+      ContainerRequest storedContainer1 = 
+          new ContainerRequest(capability, nodes, racks, priority);
+      amClient.addContainerRequest(storedContainer1);
+      assertTrue(amClient.ask.size() == 3);
+      assertTrue(amClient.release.size() == 0);
+      
+      List<String> localNodeBlacklist = new ArrayList<String>();
+      localNodeBlacklist.add(node);
+      
+      // put node in black list, so no container assignment
+      amClient.updateBlacklist(localNodeBlacklist, null);
+
+      int allocatedContainerCount = getAllocatedContainersNumber(amClient,
+        DEFAULT_ITERATION);
+      // the only node is in blacklist, so no allocation
+      assertTrue(allocatedContainerCount == 0);
+
+      // Remove node from blacklist, so get assigned with 2
+      amClient.updateBlacklist(null, localNodeBlacklist);
+      ContainerRequest storedContainer2 = 
+              new ContainerRequest(capability, nodes, racks, priority);
+      amClient.addContainerRequest(storedContainer2);
+      allocatedContainerCount = getAllocatedContainersNumber(amClient,
+          DEFAULT_ITERATION);
+      assertEquals(allocatedContainerCount, 2);
+      
+      // Test in case exception in allocate(), blacklist is kept
+      assertTrue(amClient.blacklistAdditions.isEmpty());
+      assertTrue(amClient.blacklistRemovals.isEmpty());
+      
+      // create a invalid ContainerRequest - memory value is minus
+      ContainerRequest invalidContainerRequest = 
+          new ContainerRequest(Resource.newInstance(-1024, 1),
+              nodes, racks, priority);
+      amClient.addContainerRequest(invalidContainerRequest);
+      amClient.updateBlacklist(localNodeBlacklist, null);
+      try {
+        // allocate() should complain as ContainerRequest is invalid.
+        amClient.allocate(0.1f);
+        fail("there should be an exception here.");
+      } catch (Exception e) {
+        assertEquals(amClient.blacklistAdditions.size(), 1);
+      }
+    } finally {
+      if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+        amClient.stop();
+      }
+    }
+  }
+  
+  @Test (timeout=60000)
+  public void testAMRMClientWithBlacklist() throws YarnException, IOException {
+    AMRMClientImpl<ContainerRequest> amClient = null;
+    try {
+      // start am rm client
+      amClient =
+          (AMRMClientImpl<ContainerRequest>) AMRMClient
+            .<ContainerRequest> createAMRMClient();
+      amClient.init(conf);
+      amClient.start();
+      amClient.registerApplicationMaster("Host", 10000, "");
+      String[] nodes = {"node1", "node2", "node3"};
+      
+      // Add nodes[0] and nodes[1]
+      List<String> nodeList01 = new ArrayList<String>();
+      nodeList01.add(nodes[0]);
+      nodeList01.add(nodes[1]);
+      amClient.updateBlacklist(nodeList01, null);
+      assertEquals(amClient.blacklistAdditions.size(),2);
+      assertEquals(amClient.blacklistRemovals.size(),0);
+      
+      // Add nodes[0] again, verify it is not added duplicated.
+      List<String> nodeList02 = new ArrayList<String>();
+      nodeList02.add(nodes[0]);
+      nodeList02.add(nodes[2]);
+      amClient.updateBlacklist(nodeList02, null);
+      assertEquals(amClient.blacklistAdditions.size(),3);
+      assertEquals(amClient.blacklistRemovals.size(),0);
+      
+      // Add nodes[1] and nodes[2] to removal list, 
+      // Verify addition list remove these two nodes.
+      List<String> nodeList12 = new ArrayList<String>();
+      nodeList12.add(nodes[1]);
+      nodeList12.add(nodes[2]);
+      amClient.updateBlacklist(null, nodeList12);
+      assertEquals(amClient.blacklistAdditions.size(),1);
+      assertEquals(amClient.blacklistRemovals.size(),2);
+      
+      // Add nodes[1] again to addition list, 
+      // Verify removal list will remove this node.
+      List<String> nodeList1 = new ArrayList<String>();
+      nodeList1.add(nodes[1]);
+      amClient.updateBlacklist(nodeList1, null);
+      assertEquals(amClient.blacklistAdditions.size(),2);
+      assertEquals(amClient.blacklistRemovals.size(),1);
+    } finally {
+      if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+        amClient.stop();
+      }
+    }
+  }
+
+  private int getAllocatedContainersNumber(
+      AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
+      throws YarnException, IOException {
+    int allocatedContainerCount = 0;
+    while (iterationsLeft-- > 0) {
+      Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
+      AllocateResponse allocResponse = amClient.allocate(0.1f);
+      assertTrue(amClient.ask.size() == 0);
+      assertTrue(amClient.release.size() == 0);
+        
+      assertTrue(nodeCount == amClient.getClusterNodeCount());
+      allocatedContainerCount += allocResponse.getAllocatedContainers().size();
+        
+      if(allocatedContainerCount == 0) {
+        // sleep to let NM's heartbeat to RM and trigger allocations
+        sleep(100);
+      }
+    }
+    return allocatedContainerCount;
+  }
 
   @Test (timeout=60000)
   public void testAMRMClient() throws YarnException, IOException {

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Sep  3 23:20:45 2013
@@ -86,6 +86,7 @@ public class TestYarnClient {
     client.init(conf);
     client.start();
     client.stop();
+    rm.stop();
   }
 
   @Test (timeout = 30000)

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Tue Sep  3 23:20:45 2013
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.when;
 import static org.mockito.Mockito.doThrow;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.util.ArrayList;
@@ -72,6 +73,7 @@ public class TestYarnCLI {
     sysOut = spy(new PrintStream(sysOutStream));
     sysErrStream = new ByteArrayOutputStream();
     sysErr = spy(new PrintStream(sysErrStream));
+    System.setOut(sysOut);
   }
   
   @Test
@@ -167,15 +169,35 @@ public class TestYarnCLI {
         null);
     applicationReports.add(newApplicationReport4);
 
+    ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
+    ApplicationReport newApplicationReport5 = ApplicationReport.newInstance(
+        applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5),
+        "user5", "queue5", "appname5", "host5", 128, null,
+        YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
+        FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
+        null);
+    applicationReports.add(newApplicationReport5);
+
+    ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
+    ApplicationReport newApplicationReport6 = ApplicationReport.newInstance(
+        applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6),
+        "user6", "queue6", "appname6", "host6", 129, null,
+        YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
+        FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
+        null);
+    applicationReports.add(newApplicationReport6);
+
     // Test command yarn application -list
     // if the set appStates is empty, RUNNING state will be automatically added
     // to the appStates list
     // the output of yarn application -list should be the same as
-    // equals to yarn application -list --appStates RUNNING
+    // equals to yarn application -list --appStates RUNNING,ACCEPTED,SUBMITTED
     Set<String> appType1 = new HashSet<String>();
     EnumSet<YarnApplicationState> appState1 =
         EnumSet.noneOf(YarnApplicationState.class);
     appState1.add(YarnApplicationState.RUNNING);
+    appState1.add(YarnApplicationState.ACCEPTED);
+    appState1.add(YarnApplicationState.SUBMITTED);
     when(client.getApplications(appType1, appState1)).thenReturn(
         getApplicationReports(applicationReports, appType1, appState1, false));
     int result = cli.run(new String[] { "-list" });
@@ -185,7 +207,7 @@ public class TestYarnCLI {
     ByteArrayOutputStream baos = new ByteArrayOutputStream();
     PrintWriter pw = new PrintWriter(baos);
     pw.println("Total number of applications (application-types: " + appType1
-        + " and states: " + appState1 + ")" + ":" + 2);
+        + " and states: " + appState1 + ")" + ":" + 4);
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t    Application-Type");
     pw.print("\t      User\t     Queue\t             State\t       ");
@@ -201,6 +223,16 @@ public class TestYarnCLI {
     pw.print("queue3\t           RUNNING\t         ");
     pw.print("SUCCEEDED\t         73.79%");
     pw.println("\t                                N/A");
+    pw.print("         application_1234_0009\t            ");
+    pw.print("appname5\t                HIVE\t     user5\t    ");
+    pw.print("queue5\t          ACCEPTED\t            ");
+    pw.print("KILLED\t         93.79%");
+    pw.println("\t                                N/A");
+    pw.print("         application_1234_0010\t            ");
+    pw.print("appname6\t                 PIG\t     user6\t    ");
+    pw.print("queue6\t         SUBMITTED\t            ");
+    pw.print("KILLED\t         99.79%");
+    pw.println("\t                                N/A");
     pw.close();
     String appsReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appsReportStr, sysOutStream.toString());
@@ -208,7 +240,8 @@ public class TestYarnCLI {
 
     //Test command yarn application -list --appTypes apptype1,apptype2
     //the output should be the same as
-    //yarn application -list --appTypes apptyp1, apptype2 --appStates RUNNING
+    // yarn application -list --appTypes apptyp1, apptype2 --appStates
+    // RUNNING,ACCEPTED,SUBMITTED
     sysOutStream.reset();
     Set<String> appType2 = new HashSet<String>();
     appType2.add("YARN");
@@ -217,6 +250,8 @@ public class TestYarnCLI {
     EnumSet<YarnApplicationState> appState2 =
         EnumSet.noneOf(YarnApplicationState.class);
     appState2.add(YarnApplicationState.RUNNING);
+    appState2.add(YarnApplicationState.ACCEPTED);
+    appState2.add(YarnApplicationState.SUBMITTED);
     when(client.getApplications(appType2, appState2)).thenReturn(
         getApplicationReports(applicationReports, appType2, appState2, false));
     result =
@@ -358,7 +393,7 @@ public class TestYarnCLI {
     baos = new ByteArrayOutputStream();
     pw = new PrintWriter(baos);
     pw.println("Total number of applications (application-types: " + appType5
-        + " and states: " + appState5 + ")" + ":" + 4);
+        + " and states: " + appState5 + ")" + ":" + 6);
     pw.print("                Application-Id\t    Application-Name");
     pw.print("\t    Application-Type");
     pw.print("\t      User\t     Queue\t             State\t       ");
@@ -384,6 +419,16 @@ public class TestYarnCLI {
     pw.print("queue4\t            FAILED\t         ");
     pw.print("SUCCEEDED\t         83.79%");
     pw.println("\t                                N/A");
+    pw.print("         application_1234_0009\t            ");
+    pw.print("appname5\t                HIVE\t     user5\t    ");
+    pw.print("queue5\t          ACCEPTED\t            ");
+    pw.print("KILLED\t         93.79%");
+    pw.println("\t                                N/A");
+    pw.print("         application_1234_0010\t            ");
+    pw.print("appname6\t                 PIG\t     user6\t    ");
+    pw.print("queue6\t         SUBMITTED\t            ");
+    pw.print("KILLED\t         99.79%");
+    pw.println("\t                                N/A");
     pw.close();
     appsReportStr = baos.toString("UTF-8");
     Assert.assertEquals(appsReportStr, sysOutStream.toString());
@@ -456,21 +501,40 @@ public class TestYarnCLI {
   }
 
   @Test (timeout = 10000)
-  public void testHelpCommand() throws Exception {
+  public void testAppsHelpCommand() throws Exception {
     ApplicationCLI cli = createAndGetAppCLI();
     ApplicationCLI spyCli = spy(cli);
     int result = spyCli.run(new String[] { "-help" });
     Assert.assertTrue(result == 0);
     verify(spyCli).printUsage(any(Options.class));
+    Assert.assertEquals(createApplicationCLIHelpMessage(),
+        sysOutStream.toString());
 
+    sysOutStream.reset();
     ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
     result =
         cli.run(new String[] { "-kill", applicationId.toString(), "args" });
     verify(spyCli).printUsage(any(Options.class));
+    Assert.assertEquals(createApplicationCLIHelpMessage(),
+        sysOutStream.toString());
 
+    sysOutStream.reset();
     NodeId nodeId = NodeId.newInstance("host0", 0);
     result = cli.run(new String[] { "-status", nodeId.toString(), "args" });
     verify(spyCli).printUsage(any(Options.class));
+    Assert.assertEquals(createApplicationCLIHelpMessage(),
+        sysOutStream.toString());
+  }
+
+  @Test (timeout = 5000)
+  public void testNodesHelpCommand() throws Exception {
+    NodeCLI nodeCLI = new NodeCLI();
+    nodeCLI.setClient(client);
+    nodeCLI.setSysOutPrintStream(sysOut);
+    nodeCLI.setSysErrPrintStream(sysErr);
+    nodeCLI.run(new String[] {});
+    Assert.assertEquals(createNodeCLIHelpMessage(),
+        sysOutStream.toString());
   }
 
   @Test
@@ -806,6 +870,25 @@ public class TestYarnCLI {
     verifyUsageInfo(new NodeCLI());
   }
 
+  @Test
+  public void testMissingArguments() throws Exception {
+    ApplicationCLI cli = createAndGetAppCLI();
+    int result = cli.run(new String[] { "-status" });
+    Assert.assertEquals(result, -1);
+    Assert.assertEquals("Missing argument for options\n"
+        + createApplicationCLIHelpMessage(), sysOutStream.toString());
+
+    sysOutStream.reset();
+    NodeCLI nodeCLI = new NodeCLI();
+    nodeCLI.setClient(client);
+    nodeCLI.setSysOutPrintStream(sysOut);
+    nodeCLI.setSysErrPrintStream(sysErr);
+    result = nodeCLI.run(new String[] { "-status" });
+    Assert.assertEquals(result, -1);
+    Assert.assertEquals("Missing argument for options\n"
+        + createNodeCLIHelpMessage(), sysOutStream.toString());
+  }
+
   private void verifyUsageInfo(YarnCLI cli) throws Exception {
     cli.setSysErrPrintStream(sysErr);
     cli.run(new String[0]);
@@ -832,4 +915,45 @@ public class TestYarnCLI {
     return cli;
   }
 
+  private String createApplicationCLIHelpMessage() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    pw.println("usage: application");
+    pw.println(" -appStates <States>        Works with -list to filter applications based");
+    pw.println("                            on input comma-separated list of application");
+    pw.println("                            states. The valid application state can be one");
+    pw.println("                            of the following:");
+    pw.println("                            ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING,");
+    pw.println("                            FINISHED,FAILED,KILLED");
+    pw.println(" -appTypes <Types>          Works with -list to filter applications based");
+    pw.println("                            on input comma-separated list of application");
+    pw.println("                            types.");
+    pw.println(" -help                      Displays help for all commands.");
+    pw.println(" -kill <Application ID>     Kills the application.");
+    pw.println(" -list                      List applications from the RM. Supports");
+    pw.println("                            optional use of -appTypes to filter");
+    pw.println("                            applications based on application type, and");
+    pw.println("                            -appStates to filter applications based on");
+    pw.println("                            application state");
+    pw.println(" -status <Application ID>   Prints the status of the application.");
+    pw.close();
+    String appsHelpStr = baos.toString("UTF-8");
+    return appsHelpStr;
+  }
+
+  private String createNodeCLIHelpMessage() throws IOException {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    PrintWriter pw = new PrintWriter(baos);
+    pw.println("usage: node");
+    pw.println(" -all               Works with -list to list all nodes.");
+    pw.println(" -list              List all running nodes. Supports optional use of");
+    pw.println("                    -states to filter nodes based on node state, all -all");
+    pw.println("                    to list all nodes.");
+    pw.println(" -states <States>   Works with -list to filter nodes based on input");
+    pw.println("                    comma-separated list of node states.");
+    pw.println(" -status <NodeId>   Prints the status report of the node.");
+    pw.close();
+    String nodesHelpStr = baos.toString("UTF-8");
+    return nodesHelpStr;
+  }
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java Tue Sep  3 23:20:45 2013
@@ -125,7 +125,7 @@ public class ResourceBlacklistRequestPBI
 
   @Override
   public void setBlacklistAdditions(List<String> resourceNames) {
-    if (resourceNames == null) {
+    if (resourceNames == null || resourceNames.isEmpty()) {
       if (this.blacklistAdditions != null) {
         this.blacklistAdditions.clear();
       }
@@ -144,7 +144,7 @@ public class ResourceBlacklistRequestPBI
 
   @Override
   public void setBlacklistRemovals(List<String> resourceNames) {
-    if (resourceNames == null) {
+    if (resourceNames == null || resourceNames.isEmpty()) {
       if (this.blacklistRemovals != null) {
         this.blacklistRemovals.clear();
       }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java Tue Sep  3 23:20:45 2013
@@ -39,6 +39,7 @@ public class ClientToAMTokenIdentifier e
   public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN");
 
   private ApplicationAttemptId applicationAttemptId;
+  private Text clientName = new Text();
 
   // TODO: Add more information in the tokenID such that it is not
   // transferrable, more secure etc.
@@ -46,21 +47,27 @@ public class ClientToAMTokenIdentifier e
   public ClientToAMTokenIdentifier() {
   }
 
-  public ClientToAMTokenIdentifier(ApplicationAttemptId id) {
+  public ClientToAMTokenIdentifier(ApplicationAttemptId id, String client) {
     this();
     this.applicationAttemptId = id;
+    this.clientName = new Text(client);
   }
 
   public ApplicationAttemptId getApplicationAttemptID() {
     return this.applicationAttemptId;
   }
 
+  public String getClientName() {
+    return this.clientName.toString();
+  }
+
   @Override
   public void write(DataOutput out) throws IOException {
     out.writeLong(this.applicationAttemptId.getApplicationId()
       .getClusterTimestamp());
     out.writeInt(this.applicationAttemptId.getApplicationId().getId());
     out.writeInt(this.applicationAttemptId.getAttemptId());
+    this.clientName.write(out);
   }
 
   @Override
@@ -68,6 +75,7 @@ public class ClientToAMTokenIdentifier e
     this.applicationAttemptId =
         ApplicationAttemptId.newInstance(
           ApplicationId.newInstance(in.readLong(), in.readInt()), in.readInt());
+    this.clientName.readFields(in);
   }
 
   @Override
@@ -77,10 +85,10 @@ public class ClientToAMTokenIdentifier e
 
   @Override
   public UserGroupInformation getUser() {
-    if (this.applicationAttemptId == null) {
+    if (this.clientName == null) {
       return null;
     }
-    return UserGroupInformation.createRemoteUser(this.applicationAttemptId.toString());
+    return UserGroupInformation.createRemoteUser(this.clientName.toString());
   }
 
   @InterfaceAudience.Private

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java Tue Sep  3 23:20:45 2013
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.record
 public class ClientToAMTokenSecretManager extends
     BaseClientToAMTokenSecretManager {
 
-  // Only one client-token and one master-key for AM
+  // Only one master-key for AM
   private SecretKey masterKey;
 
   public ClientToAMTokenSecretManager(
@@ -53,7 +53,7 @@ public class ClientToAMTokenSecretManage
 
   @Override
   public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) {
-    // Only one client-token and one master-key for AM, just return that.
+    // Only one master-key for AM, just return that.
     return this.masterKey;
   }
 

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Tue Sep  3 23:20:45 2013
@@ -198,7 +198,8 @@ public class ConverterUtils {
     Iterator<String> it = _split(appIdStr).iterator();
     if (!it.next().equals(APPLICATION_PREFIX)) {
       throw new IllegalArgumentException("Invalid ApplicationId prefix: "
-          + appIdStr);
+          + appIdStr + ". The valid ApplicationId should start with prefix "
+          + APPLICATION_PREFIX);
     }
     try {
       return toApplicationId(it);

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Tue Sep  3 23:20:45 2013
@@ -242,9 +242,8 @@ public class WebApps {
         for(Map.Entry<String, Object> entry : attributes.entrySet()) {
           server.setAttribute(entry.getKey(), entry.getValue());
         }
-        String webAppPath = "/" + name + "/*";
         server.defineFilter(server.getWebAppContext(), "guice",
-          GuiceFilter.class.getName(), null, new String[] { webAppPath, "/" });
+          GuiceFilter.class.getName(), null, new String[] { "/*" });
 
         webapp.setConf(conf);
         webapp.setHttpServer(server);

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java Tue Sep  3 23:20:45 2013
@@ -19,11 +19,14 @@
 package org.apache.hadoop.yarn.util;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
 import org.apache.hadoop.service.BreakableService;
 import org.apache.hadoop.service.CompositeService;
 import org.apache.hadoop.service.Service;
@@ -314,6 +317,26 @@ public class TestCompositeService {
     composite.init(new Configuration());
     assertInState(STATE.INITED, child);
   }
+
+  @Test (timeout = 1000)
+  public void testAddIfService() {
+    CompositeService testService = new CompositeService("TestService") {
+      Service service;
+      @Override
+      public void serviceInit(Configuration conf) {
+        Integer notAService = new Integer(0);
+        assertFalse("Added an integer as a service",
+            addIfService(notAService));
+
+        service = new AbstractService("Service") {};
+        assertTrue("Unable to add a service", addIfService(service));
+      }
+    };
+
+    testService.init(new Configuration());
+    assertEquals("Incorrect number of services",
+        1, testService.getServices().size());
+  }
   
   public static class CompositeServiceAddingAChild extends CompositeService{
     Service child;

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java Tue Sep  3 23:20:45 2013
@@ -18,30 +18,47 @@
 
 package org.apache.hadoop.yarn.webapp;
 
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebApps;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage;
-import org.apache.hadoop.yarn.webapp.view.JQueryUI;
-import org.apache.hadoop.yarn.webapp.view.TextPage;
-
-import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
 
 import java.io.InputStream;
 import java.net.HttpURLConnection;
 import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import javax.xml.bind.JAXBContext;
 
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
-
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.webapp.view.HtmlPage;
+import org.apache.hadoop.yarn.webapp.view.JQueryUI;
+import org.apache.hadoop.yarn.webapp.view.TextPage;
 import org.junit.Test;
-import static org.junit.Assert.*;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+
 public class TestWebApp {
   static final Logger LOG = LoggerFactory.getLogger(TestWebApp.class);
 
@@ -227,14 +244,19 @@ public class TestWebApp {
   }
 
   @Test public void testCustomRoutes() throws Exception {
-    WebApp app = WebApps.$for("test", this).start(new WebApp() {
-      @Override public void setup() {
-        route("/:foo", FooController.class);
-        route("/bar/foo", FooController.class, "bar");
-        route("/foo/:foo", DefaultController.class);
-        route("/foo/bar/:foo", DefaultController.class, "index");
-      }
-    });
+    WebApp app =
+        WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
+          @Override
+          public void setup() {
+            bind(MyTestJAXBContextResolver.class);
+            bind(MyTestWebService.class);
+
+            route("/:foo", FooController.class);
+            route("/bar/foo", FooController.class, "bar");
+            route("/foo/:foo", DefaultController.class);
+            route("/foo/bar/:foo", DefaultController.class, "index");
+          }
+        });
     String baseUrl = baseUrl(app);
     try {
       assertEquals("foo", getContent(baseUrl).trim());
@@ -245,6 +267,31 @@ public class TestWebApp {
       assertEquals("default1", getContent(baseUrl +"test/foo/1").trim());
       assertEquals("default2", getContent(baseUrl +"test/foo/bar/2").trim());
       assertEquals(404, getResponseCode(baseUrl +"test/goo"));
+      assertEquals(200, getResponseCode(baseUrl +"ws/v1/test"));
+      assertTrue(getContent(baseUrl +"ws/v1/test").contains("myInfo"));
+    } finally {
+      app.stop();
+    }
+  }
+
+  // This is to test the GuiceFilter should only be applied to webAppContext,
+  // not to staticContext  and logContext;
+  @Test public void testYARNWebAppContext() throws Exception {
+    // setting up the log context
+    System.setProperty("hadoop.log.dir", "/Not/Existing/dir");
+    WebApp app = WebApps.$for("test", this).start(new WebApp() {
+      @Override public void setup() {
+        route("/", FooController.class);
+      }
+    });
+    String baseUrl = baseUrl(app);
+    try {
+      // should not redirect to foo
+      assertFalse("foo".equals(getContent(baseUrl +"static").trim()));
+      // Not able to access a non-existing dir, should not redirect to foo.
+      assertEquals(404, getResponseCode(baseUrl +"logs"));
+      // should be able to redirect to foo.
+      assertEquals("foo", getContent(baseUrl).trim());
     } finally {
       app.stop();
     }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Tue Sep  3 23:20:45 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 
 /**
  * Context interface for sharing information across components in the
@@ -61,4 +62,8 @@ public interface Context {
   NodeHealthStatus getNodeHealthStatus();
 
   ContainerManagementProtocol getContainerManager();
+
+  LocalDirsHandlerService getLocalDirsHandler();
+
+  ApplicationACLsManager getApplicationACLsManager();
 }

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Sep  3 23:20:45 2013
@@ -123,7 +123,8 @@ public class NodeManager extends Composi
   protected NMContext createNMContext(
       NMContainerTokenSecretManager containerTokenSecretManager,
       NMTokenSecretManagerInNM nmTokenSecretManager) {
-    return new NMContext(containerTokenSecretManager, nmTokenSecretManager);
+    return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
+        dirsHandler, aclsManager);
   }
 
   protected void doSecureLogin() throws IOException {
@@ -142,9 +143,6 @@ public class NodeManager extends Composi
     NMTokenSecretManagerInNM nmTokenSecretManager =
         new NMTokenSecretManagerInNM();
     
-    this.context =
-        createNMContext(containerTokenSecretManager, nmTokenSecretManager);
-
     this.aclsManager = new ApplicationACLsManager(conf);
 
     ContainerExecutor exec = ReflectionUtils.newInstance(
@@ -165,7 +163,9 @@ public class NodeManager extends Composi
     addService(nodeHealthChecker);
     dirsHandler = nodeHealthChecker.getDiskHandler();
 
-
+    this.context = createNMContext(containerTokenSecretManager,
+        nmTokenSecretManager);
+    
     nodeStatusUpdater =
         createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
 
@@ -319,14 +319,19 @@ public class NodeManager extends Composi
     private final NMContainerTokenSecretManager containerTokenSecretManager;
     private final NMTokenSecretManagerInNM nmTokenSecretManager;
     private ContainerManagementProtocol containerManager;
+    private final LocalDirsHandlerService dirsHandler;
+    private final ApplicationACLsManager aclsManager;
     private WebServer webServer;
     private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
         .getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
-
+        
     public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
-        NMTokenSecretManagerInNM nmTokenSecretManager) {
+        NMTokenSecretManagerInNM nmTokenSecretManager,
+        LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager) {
       this.containerTokenSecretManager = containerTokenSecretManager;
       this.nmTokenSecretManager = nmTokenSecretManager;
+      this.dirsHandler = dirsHandler;
+      this.aclsManager = aclsManager;
       this.nodeHealthStatus.setIsNodeHealthy(true);
       this.nodeHealthStatus.setHealthReport("Healthy");
       this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
@@ -386,6 +391,16 @@ public class NodeManager extends Composi
     public void setNodeId(NodeId nodeId) {
       this.nodeId = nodeId;
     }
+
+    @Override
+    public LocalDirsHandlerService getLocalDirsHandler() {
+      return dirsHandler;
+    }
+    
+    @Override
+    public ApplicationACLsManager getApplicationACLsManager() {
+      return aclsManager;
+    }
   }
 
 

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Sep  3 23:20:45 2013
@@ -85,7 +85,6 @@ import org.apache.hadoop.yarn.server.nod
 import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
 import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
 import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -193,12 +192,6 @@ public class ContainerManagerImpl extend
     super.serviceInit(conf);
   }
 
-  private void addIfService(Object object) {
-    if (object instanceof Service) {
-      addService((Service) object);
-    }
-  }
-
   protected LogHandler createLogHandler(Configuration conf, Context context,
       DeletionService deletionService) {
     if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
@@ -475,8 +468,7 @@ public class ContainerManagerImpl extend
 
     // Create the application
     Application application =
-        new ApplicationImpl(dispatcher, this.aclsManager, user, applicationID,
-          credentials, context);
+        new ApplicationImpl(dispatcher, user, applicationID, credentials, context);
     if (null == context.getApplications().putIfAbsent(applicationID,
       application)) {
       LOG.info("Creating a new application reference for app " + applicationID);

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Sep  3 23:20:45 2013
@@ -73,14 +73,13 @@ public class ApplicationImpl implements 
   Map<ContainerId, Container> containers =
       new HashMap<ContainerId, Container>();
 
-  public ApplicationImpl(Dispatcher dispatcher,
-      ApplicationACLsManager aclsManager, String user, ApplicationId appId,
+  public ApplicationImpl(Dispatcher dispatcher, String user, ApplicationId appId,
       Credentials credentials, Context context) {
     this.dispatcher = dispatcher;
     this.user = user;
     this.appId = appId;
     this.credentials = credentials;
-    this.aclsManager = aclsManager;
+    this.aclsManager = context.getApplicationACLsManager();
     this.context = context;
     ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
     readLock = lock.readLock();

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Tue Sep  3 23:20:45 2013
@@ -525,7 +525,8 @@ public class ContainerLaunch implements 
 
     @Override
     public void env(String key, String value) {
-      line("@set ", key, "=", value);
+      line("@set ", key, "=", value,
+          "\nif %errorlevel% neq 0 exit /b %errorlevel%");
     }
 
     @Override

Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Tue Sep  3 23:20:45 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
 import org.apache.hadoop.fs.FileContext;
 import org.apache.hadoop.fs.UnsupportedFileSystemException;
 import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
@@ -149,7 +150,8 @@ public class ContainersLauncher extends 
               dispatcher.getEventHandler().handle(
                   new ContainerExitEvent(containerId,
                       ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
-                      ExitCode.TERMINATED.getExitCode(),
+                      Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() : 
+                        ExitCode.TERMINATED.getExitCode(),
                       "Container terminated before launch."));
             }
           }