You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2013/09/04 01:20:48 UTC
svn commit: r1519885 [1/3] - in
/hadoop/common/branches/HDFS-2832/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/
hadoop-yarn/...
Author: arp
Date: Tue Sep 3 23:20:45 2013
New Revision: 1519885
URL: http://svn.apache.org/r1519885
Log:
Merging r1518852 through r1519883 from trunk to branch HDFS-2832
Added:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
- copied unchanged from r1519883, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestJAXBContextResolver.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
- copied unchanged from r1519883, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/MyTestWebService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
- copied unchanged from r1519883, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Tue Sep 3 23:20:45 2013
@@ -21,6 +21,9 @@ Release 2.3.0 - UNRELEASED
NEW FEATURES
+ YARN-649. Added a new NM web-service to serve container logs in plain text
+ over HTTP. (Sandy Ryza via vinodkv)
+
IMPROVEMENTS
YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
@@ -39,6 +42,9 @@ Release 2.1.1-beta - UNRELEASED
INCOMPATIBLE CHANGES
+ YARN-707. Added user information also in the YARN ClientToken so that AMs
+ can implement authorization based on incoming users. (Jason Lowe via vinodkv)
+
NEW FEATURES
IMPROVEMENTS
@@ -66,6 +72,20 @@ Release 2.1.1-beta - UNRELEASED
YARN-1080. Improved help message for "yarn logs" command. (Xuan Gong via
vinodkv)
+ YARN-771. AMRMClient support for resource blacklisting (Junping Du via
+ bikas)
+
+ YARN-1117. Improved help messages for "yarn application" and "yarn node"
+ commands. (Xuan Gong via vinodkv)
+
+ YARN-1120. Made ApplicationConstants.Environment.USER definition OS neutral
+ as the corresponding value is now set correctly end-to-end. (Chuan Liu via
+ vinodkv)
+
+ YARN-1124. Modified YARN CLI application list to display new and submitted
+ applications together with running apps by default, following up YARN-1074.
+ (Xuan Gong via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -126,6 +146,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-1101. Active nodes can be decremented below 0 (Robert Parker
via tgraves)
+ YARN-1077. Fixed TestContainerLaunch test failure on Windows. (Chuan Liu via
+ vinodkv)
+
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Tue Sep 3 23:20:45 2013
@@ -80,7 +80,7 @@ public interface ApplicationConstants {
* $USER
* Final, non-modifiable.
*/
- USER(Shell.WINDOWS ? "USERNAME": "USER"),
+ USER("USER"),
/**
* $LOGNAME
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java Tue Sep 3 23:20:45 2013
@@ -286,4 +286,15 @@ public abstract class AMRMClient<T exten
Priority priority,
String resourceName,
Resource capability);
+
+ /**
+ * Update application's blacklist with addition or removal resources.
+ *
+ * @param blacklistAdditions list of resources which should be added to the
+ * application blacklist
+ * @param blacklistRemovals list of resources which should be removed from the
+ * application blacklist
+ */
+ public abstract void updateBlacklist(List<String> blacklistAdditions,
+ List<String> blacklistRemovals);
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java Tue Sep 3 23:20:45 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -80,6 +81,9 @@ public class AMRMClientImpl<T extends Co
protected Resource clusterAvailableResources;
protected int clusterNodeCount;
+ protected final Set<String> blacklistAdditions = new HashSet<String>();
+ protected final Set<String> blacklistRemovals = new HashSet<String>();
+
class ResourceRequestInfo {
ResourceRequest remoteRequest;
LinkedHashSet<T> containerRequests;
@@ -199,9 +203,11 @@ public class AMRMClientImpl<T extends Co
Preconditions.checkArgument(progressIndicator >= 0,
"Progress indicator should not be negative");
AllocateResponse allocateResponse = null;
- ArrayList<ResourceRequest> askList = null;
- ArrayList<ContainerId> releaseList = null;
+ List<ResourceRequest> askList = null;
+ List<ContainerId> releaseList = null;
AllocateRequest allocateRequest = null;
+ List<String> blacklistToAdd = new ArrayList<String>();
+ List<String> blacklistToRemove = new ArrayList<String>();
try {
synchronized (this) {
@@ -217,9 +223,22 @@ public class AMRMClientImpl<T extends Co
// optimistically clear this collection assuming no RPC failure
ask.clear();
release.clear();
+
+ blacklistToAdd.addAll(blacklistAdditions);
+ blacklistToRemove.addAll(blacklistRemovals);
+
+ ResourceBlacklistRequest blacklistRequest =
+ (blacklistToAdd != null) || (blacklistToRemove != null) ?
+ ResourceBlacklistRequest.newInstance(blacklistToAdd,
+ blacklistToRemove) : null;
+
allocateRequest =
AllocateRequest.newInstance(lastResponseId, progressIndicator,
- askList, releaseList, null);
+ askList, releaseList, blacklistRequest);
+ // clear blacklistAdditions and blacklistRemovals before
+ // unsynchronized part
+ blacklistAdditions.clear();
+ blacklistRemovals.clear();
}
allocateResponse = rmClient.allocate(allocateRequest);
@@ -253,6 +272,9 @@ public class AMRMClientImpl<T extends Co
ask.add(oldAsk);
}
}
+
+ blacklistAdditions.addAll(blacklistToAdd);
+ blacklistRemovals.addAll(blacklistToRemove);
}
}
}
@@ -604,4 +626,31 @@ public class AMRMClientImpl<T extends Co
+ " #asks=" + ask.size());
}
}
+
+ @Override
+ public synchronized void updateBlacklist(List<String> blacklistAdditions,
+ List<String> blacklistRemovals) {
+
+ if (blacklistAdditions != null) {
+ this.blacklistAdditions.addAll(blacklistAdditions);
+ // if some resources are also in blacklistRemovals updated before, we
+ // should remove them here.
+ this.blacklistRemovals.removeAll(blacklistAdditions);
+ }
+
+ if (blacklistRemovals != null) {
+ this.blacklistRemovals.addAll(blacklistRemovals);
+ // if some resources are in blacklistAdditions before, we should remove
+ // them here.
+ this.blacklistAdditions.removeAll(blacklistRemovals);
+ }
+
+ if (blacklistAdditions != null && blacklistRemovals != null
+ && blacklistAdditions.removeAll(blacklistRemovals)) {
+ // we allow resources to appear in addition list and removal list in the
+ // same invocation of updateBlacklist(), but should get a warn here.
+ LOG.warn("The same resources appear in both blacklistAdditions and " +
+ "blacklistRemovals in updateBlacklist.");
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java Tue Sep 3 23:20:45 2013
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -70,32 +71,38 @@ public class ApplicationCLI extends Yarn
Options opts = new Options();
opts.addOption(STATUS_CMD, true, "Prints the status of the application.");
opts.addOption(LIST_CMD, false, "List applications from the RM. " +
- "Supports optional use of --appTypes to filter applications " +
+ "Supports optional use of -appTypes to filter applications " +
"based on application type, " +
- "and --appStates to filter applications based on application state");
+ "and -appStates to filter applications based on application state");
opts.addOption(KILL_CMD, true, "Kills the application.");
opts.addOption(HELP_CMD, false, "Displays help for all commands.");
- Option appTypeOpt = new Option(APP_TYPE_CMD, true,
- "Works with --list to filter applications based on their type.");
+ Option appTypeOpt = new Option(APP_TYPE_CMD, true, "Works with -list to " +
+ "filter applications based on " +
+ "input comma-separated list of application types.");
appTypeOpt.setValueSeparator(',');
appTypeOpt.setArgs(Option.UNLIMITED_VALUES);
- appTypeOpt.setArgName("Comma-separated list of application types");
+ appTypeOpt.setArgName("Types");
opts.addOption(appTypeOpt);
- Option appStateOpt =
- new Option(
- APP_STATE_CMD,
- true,
- "Works with --list to filter applications based on their state. "
- + getAllValidApplicationStates());
+ Option appStateOpt = new Option(APP_STATE_CMD, true, "Works with -list " +
+ "to filter applications based on input comma-separated list of " +
+ "application states. " + getAllValidApplicationStates());
appStateOpt.setValueSeparator(',');
appStateOpt.setArgs(Option.UNLIMITED_VALUES);
- appStateOpt.setArgName("Comma-separated list of application states");
+ appStateOpt.setArgName("States");
opts.addOption(appStateOpt);
opts.getOption(KILL_CMD).setArgName("Application ID");
opts.getOption(STATUS_CMD).setArgName("Application ID");
- CommandLine cliParser = new GnuParser().parse(opts, args);
int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ sysout.println("Missing argument for options");
+ printUsage(opts);
+ return exitCode;
+ }
+
if (cliParser.hasOption(STATUS_CMD)) {
if (args.length != 2) {
printUsage(opts);
@@ -187,6 +194,8 @@ public class ApplicationCLI extends Yarn
} else {
if (appStates.isEmpty()) {
appStates.add(YarnApplicationState.RUNNING);
+ appStates.add(YarnApplicationState.ACCEPTED);
+ appStates.add(YarnApplicationState.SUBMITTED);
}
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java Tue Sep 3 23:20:45 2013
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.MissingArgumentException;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
@@ -64,20 +65,29 @@ public class NodeCLI extends YarnCLI {
Options opts = new Options();
opts.addOption(STATUS_CMD, true, "Prints the status report of the node.");
opts.addOption(LIST_CMD, false, "List all running nodes. " +
- "Supports optional use of --states to filter nodes " +
- "based on node state, all --all to list all nodes.");
+ "Supports optional use of -states to filter nodes " +
+ "based on node state, all -all to list all nodes.");
Option nodeStateOpt = new Option(NODE_STATE_CMD, true,
- "Works with -list to filter nodes based on their states.");
+ "Works with -list to filter nodes based on input comma-separated list of node states.");
nodeStateOpt.setValueSeparator(',');
nodeStateOpt.setArgs(Option.UNLIMITED_VALUES);
- nodeStateOpt.setArgName("Comma-separated list of node states");
+ nodeStateOpt.setArgName("States");
opts.addOption(nodeStateOpt);
Option allOpt = new Option(NODE_ALL, false,
"Works with -list to list all nodes.");
opts.addOption(allOpt);
- CommandLine cliParser = new GnuParser().parse(opts, args);
+ opts.getOption(STATUS_CMD).setArgName("NodeId");
int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ sysout.println("Missing argument for options");
+ printUsage(opts);
+ return exitCode;
+ }
+
if (cliParser.hasOption("status")) {
if (args.length != 2) {
printUsage(opts);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Tue Sep 3 23:20:45 2013
@@ -18,13 +18,16 @@
package org.apache.hadoop.yarn.client.api.impl;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -97,6 +100,7 @@ public class TestAMRMClient {
static String rack;
static String[] nodes;
static String[] racks;
+ private final static int DEFAULT_ITERATION = 3;
@BeforeClass
public static void setup() throws Exception {
@@ -476,6 +480,144 @@ public class TestAMRMClient {
}
}
}
+
+ @Test (timeout=60000)
+ public void testAllocationWithBlacklist() throws YarnException, IOException {
+ AMRMClientImpl<ContainerRequest> amClient = null;
+ try {
+ // start am rm client
+ amClient =
+ (AMRMClientImpl<ContainerRequest>) AMRMClient
+ .<ContainerRequest> createAMRMClient();
+ amClient.init(conf);
+ amClient.start();
+ amClient.registerApplicationMaster("Host", 10000, "");
+
+ assertTrue(amClient.ask.size() == 0);
+ assertTrue(amClient.release.size() == 0);
+
+ ContainerRequest storedContainer1 =
+ new ContainerRequest(capability, nodes, racks, priority);
+ amClient.addContainerRequest(storedContainer1);
+ assertTrue(amClient.ask.size() == 3);
+ assertTrue(amClient.release.size() == 0);
+
+ List<String> localNodeBlacklist = new ArrayList<String>();
+ localNodeBlacklist.add(node);
+
+ // put node in black list, so no container assignment
+ amClient.updateBlacklist(localNodeBlacklist, null);
+
+ int allocatedContainerCount = getAllocatedContainersNumber(amClient,
+ DEFAULT_ITERATION);
+ // the only node is in blacklist, so no allocation
+ assertTrue(allocatedContainerCount == 0);
+
+ // Remove node from blacklist, so get assigned with 2
+ amClient.updateBlacklist(null, localNodeBlacklist);
+ ContainerRequest storedContainer2 =
+ new ContainerRequest(capability, nodes, racks, priority);
+ amClient.addContainerRequest(storedContainer2);
+ allocatedContainerCount = getAllocatedContainersNumber(amClient,
+ DEFAULT_ITERATION);
+ assertEquals(allocatedContainerCount, 2);
+
+ // Test in case exception in allocate(), blacklist is kept
+ assertTrue(amClient.blacklistAdditions.isEmpty());
+ assertTrue(amClient.blacklistRemovals.isEmpty());
+
+ // create a invalid ContainerRequest - memory value is minus
+ ContainerRequest invalidContainerRequest =
+ new ContainerRequest(Resource.newInstance(-1024, 1),
+ nodes, racks, priority);
+ amClient.addContainerRequest(invalidContainerRequest);
+ amClient.updateBlacklist(localNodeBlacklist, null);
+ try {
+ // allocate() should complain as ContainerRequest is invalid.
+ amClient.allocate(0.1f);
+ fail("there should be an exception here.");
+ } catch (Exception e) {
+ assertEquals(amClient.blacklistAdditions.size(), 1);
+ }
+ } finally {
+ if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+ amClient.stop();
+ }
+ }
+ }
+
+ @Test (timeout=60000)
+ public void testAMRMClientWithBlacklist() throws YarnException, IOException {
+ AMRMClientImpl<ContainerRequest> amClient = null;
+ try {
+ // start am rm client
+ amClient =
+ (AMRMClientImpl<ContainerRequest>) AMRMClient
+ .<ContainerRequest> createAMRMClient();
+ amClient.init(conf);
+ amClient.start();
+ amClient.registerApplicationMaster("Host", 10000, "");
+ String[] nodes = {"node1", "node2", "node3"};
+
+ // Add nodes[0] and nodes[1]
+ List<String> nodeList01 = new ArrayList<String>();
+ nodeList01.add(nodes[0]);
+ nodeList01.add(nodes[1]);
+ amClient.updateBlacklist(nodeList01, null);
+ assertEquals(amClient.blacklistAdditions.size(),2);
+ assertEquals(amClient.blacklistRemovals.size(),0);
+
+ // Add nodes[0] again, verify it is not added duplicated.
+ List<String> nodeList02 = new ArrayList<String>();
+ nodeList02.add(nodes[0]);
+ nodeList02.add(nodes[2]);
+ amClient.updateBlacklist(nodeList02, null);
+ assertEquals(amClient.blacklistAdditions.size(),3);
+ assertEquals(amClient.blacklistRemovals.size(),0);
+
+ // Add nodes[1] and nodes[2] to removal list,
+ // Verify addition list remove these two nodes.
+ List<String> nodeList12 = new ArrayList<String>();
+ nodeList12.add(nodes[1]);
+ nodeList12.add(nodes[2]);
+ amClient.updateBlacklist(null, nodeList12);
+ assertEquals(amClient.blacklistAdditions.size(),1);
+ assertEquals(amClient.blacklistRemovals.size(),2);
+
+ // Add nodes[1] again to addition list,
+ // Verify removal list will remove this node.
+ List<String> nodeList1 = new ArrayList<String>();
+ nodeList1.add(nodes[1]);
+ amClient.updateBlacklist(nodeList1, null);
+ assertEquals(amClient.blacklistAdditions.size(),2);
+ assertEquals(amClient.blacklistRemovals.size(),1);
+ } finally {
+ if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+ amClient.stop();
+ }
+ }
+ }
+
+ private int getAllocatedContainersNumber(
+ AMRMClientImpl<ContainerRequest> amClient, int iterationsLeft)
+ throws YarnException, IOException {
+ int allocatedContainerCount = 0;
+ while (iterationsLeft-- > 0) {
+ Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
+ AllocateResponse allocResponse = amClient.allocate(0.1f);
+ assertTrue(amClient.ask.size() == 0);
+ assertTrue(amClient.release.size() == 0);
+
+ assertTrue(nodeCount == amClient.getClusterNodeCount());
+ allocatedContainerCount += allocResponse.getAllocatedContainers().size();
+
+ if(allocatedContainerCount == 0) {
+ // sleep to let NM's heartbeat to RM and trigger allocations
+ sleep(100);
+ }
+ }
+ return allocatedContainerCount;
+ }
@Test (timeout=60000)
public void testAMRMClient() throws YarnException, IOException {
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Sep 3 23:20:45 2013
@@ -86,6 +86,7 @@ public class TestYarnClient {
client.init(conf);
client.start();
client.stop();
+ rm.stop();
}
@Test (timeout = 30000)
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Tue Sep 3 23:20:45 2013
@@ -29,6 +29,7 @@ import static org.mockito.Mockito.when;
import static org.mockito.Mockito.doThrow;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
@@ -72,6 +73,7 @@ public class TestYarnCLI {
sysOut = spy(new PrintStream(sysOutStream));
sysErrStream = new ByteArrayOutputStream();
sysErr = spy(new PrintStream(sysErrStream));
+ System.setOut(sysOut);
}
@Test
@@ -167,15 +169,35 @@ public class TestYarnCLI {
null);
applicationReports.add(newApplicationReport4);
+ ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
+ ApplicationReport newApplicationReport5 = ApplicationReport.newInstance(
+ applicationId5, ApplicationAttemptId.newInstance(applicationId5, 5),
+ "user5", "queue5", "appname5", "host5", 128, null,
+ YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
+ FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
+ null);
+ applicationReports.add(newApplicationReport5);
+
+ ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
+ ApplicationReport newApplicationReport6 = ApplicationReport.newInstance(
+ applicationId6, ApplicationAttemptId.newInstance(applicationId6, 6),
+ "user6", "queue6", "appname6", "host6", 129, null,
+ YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
+ FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
+ null);
+ applicationReports.add(newApplicationReport6);
+
// Test command yarn application -list
// if the set appStates is empty, RUNNING state will be automatically added
// to the appStates list
// the output of yarn application -list should be the same as
- // equals to yarn application -list --appStates RUNNING
+ // equals to yarn application -list --appStates RUNNING,ACCEPTED,SUBMITTED
Set<String> appType1 = new HashSet<String>();
EnumSet<YarnApplicationState> appState1 =
EnumSet.noneOf(YarnApplicationState.class);
appState1.add(YarnApplicationState.RUNNING);
+ appState1.add(YarnApplicationState.ACCEPTED);
+ appState1.add(YarnApplicationState.SUBMITTED);
when(client.getApplications(appType1, appState1)).thenReturn(
getApplicationReports(applicationReports, appType1, appState1, false));
int result = cli.run(new String[] { "-list" });
@@ -185,7 +207,7 @@ public class TestYarnCLI {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType1
- + " and states: " + appState1 + ")" + ":" + 2);
+ + " and states: " + appState1 + ")" + ":" + 4);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -201,6 +223,16 @@ public class TestYarnCLI {
pw.print("queue3\t RUNNING\t ");
pw.print("SUCCEEDED\t 73.79%");
pw.println("\t N/A");
+ pw.print(" application_1234_0009\t ");
+ pw.print("appname5\t HIVE\t user5\t ");
+ pw.print("queue5\t ACCEPTED\t ");
+ pw.print("KILLED\t 93.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0010\t ");
+ pw.print("appname6\t PIG\t user6\t ");
+ pw.print("queue6\t SUBMITTED\t ");
+ pw.print("KILLED\t 99.79%");
+ pw.println("\t N/A");
pw.close();
String appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
@@ -208,7 +240,8 @@ public class TestYarnCLI {
//Test command yarn application -list --appTypes apptype1,apptype2
//the output should be the same as
- //yarn application -list --appTypes apptyp1, apptype2 --appStates RUNNING
+ // yarn application -list --appTypes apptyp1, apptype2 --appStates
+ // RUNNING,ACCEPTED,SUBMITTED
sysOutStream.reset();
Set<String> appType2 = new HashSet<String>();
appType2.add("YARN");
@@ -217,6 +250,8 @@ public class TestYarnCLI {
EnumSet<YarnApplicationState> appState2 =
EnumSet.noneOf(YarnApplicationState.class);
appState2.add(YarnApplicationState.RUNNING);
+ appState2.add(YarnApplicationState.ACCEPTED);
+ appState2.add(YarnApplicationState.SUBMITTED);
when(client.getApplications(appType2, appState2)).thenReturn(
getApplicationReports(applicationReports, appType2, appState2, false));
result =
@@ -358,7 +393,7 @@ public class TestYarnCLI {
baos = new ByteArrayOutputStream();
pw = new PrintWriter(baos);
pw.println("Total number of applications (application-types: " + appType5
- + " and states: " + appState5 + ")" + ":" + 4);
+ + " and states: " + appState5 + ")" + ":" + 6);
pw.print(" Application-Id\t Application-Name");
pw.print("\t Application-Type");
pw.print("\t User\t Queue\t State\t ");
@@ -384,6 +419,16 @@ public class TestYarnCLI {
pw.print("queue4\t FAILED\t ");
pw.print("SUCCEEDED\t 83.79%");
pw.println("\t N/A");
+ pw.print(" application_1234_0009\t ");
+ pw.print("appname5\t HIVE\t user5\t ");
+ pw.print("queue5\t ACCEPTED\t ");
+ pw.print("KILLED\t 93.79%");
+ pw.println("\t N/A");
+ pw.print(" application_1234_0010\t ");
+ pw.print("appname6\t PIG\t user6\t ");
+ pw.print("queue6\t SUBMITTED\t ");
+ pw.print("KILLED\t 99.79%");
+ pw.println("\t N/A");
pw.close();
appsReportStr = baos.toString("UTF-8");
Assert.assertEquals(appsReportStr, sysOutStream.toString());
@@ -456,21 +501,40 @@ public class TestYarnCLI {
}
@Test (timeout = 10000)
- public void testHelpCommand() throws Exception {
+ public void testAppsHelpCommand() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationCLI spyCli = spy(cli);
int result = spyCli.run(new String[] { "-help" });
Assert.assertTrue(result == 0);
verify(spyCli).printUsage(any(Options.class));
+ Assert.assertEquals(createApplicationCLIHelpMessage(),
+ sysOutStream.toString());
+ sysOutStream.reset();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
result =
cli.run(new String[] { "-kill", applicationId.toString(), "args" });
verify(spyCli).printUsage(any(Options.class));
+ Assert.assertEquals(createApplicationCLIHelpMessage(),
+ sysOutStream.toString());
+ sysOutStream.reset();
NodeId nodeId = NodeId.newInstance("host0", 0);
result = cli.run(new String[] { "-status", nodeId.toString(), "args" });
verify(spyCli).printUsage(any(Options.class));
+ Assert.assertEquals(createApplicationCLIHelpMessage(),
+ sysOutStream.toString());
+ }
+
+ @Test (timeout = 5000)
+ public void testNodesHelpCommand() throws Exception {
+ NodeCLI nodeCLI = new NodeCLI();
+ nodeCLI.setClient(client);
+ nodeCLI.setSysOutPrintStream(sysOut);
+ nodeCLI.setSysErrPrintStream(sysErr);
+ nodeCLI.run(new String[] {});
+ Assert.assertEquals(createNodeCLIHelpMessage(),
+ sysOutStream.toString());
}
@Test
@@ -806,6 +870,25 @@ public class TestYarnCLI {
verifyUsageInfo(new NodeCLI());
}
+ @Test
+ public void testMissingArguments() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ int result = cli.run(new String[] { "-status" });
+ Assert.assertEquals(result, -1);
+ Assert.assertEquals("Missing argument for options\n"
+ + createApplicationCLIHelpMessage(), sysOutStream.toString());
+
+ sysOutStream.reset();
+ NodeCLI nodeCLI = new NodeCLI();
+ nodeCLI.setClient(client);
+ nodeCLI.setSysOutPrintStream(sysOut);
+ nodeCLI.setSysErrPrintStream(sysErr);
+ result = nodeCLI.run(new String[] { "-status" });
+ Assert.assertEquals(result, -1);
+ Assert.assertEquals("Missing argument for options\n"
+ + createNodeCLIHelpMessage(), sysOutStream.toString());
+ }
+
private void verifyUsageInfo(YarnCLI cli) throws Exception {
cli.setSysErrPrintStream(sysErr);
cli.run(new String[0]);
@@ -832,4 +915,45 @@ public class TestYarnCLI {
return cli;
}
+ private String createApplicationCLIHelpMessage() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("usage: application");
+ pw.println(" -appStates <States> Works with -list to filter applications based");
+ pw.println(" on input comma-separated list of application");
+ pw.println(" states. The valid application state can be one");
+ pw.println(" of the following:");
+ pw.println(" ALL,NEW,NEW_SAVING,SUBMITTED,ACCEPTED,RUNNING,");
+ pw.println(" FINISHED,FAILED,KILLED");
+ pw.println(" -appTypes <Types> Works with -list to filter applications based");
+ pw.println(" on input comma-separated list of application");
+ pw.println(" types.");
+ pw.println(" -help Displays help for all commands.");
+ pw.println(" -kill <Application ID> Kills the application.");
+ pw.println(" -list List applications from the RM. Supports");
+ pw.println(" optional use of -appTypes to filter");
+ pw.println(" applications based on application type, and");
+ pw.println(" -appStates to filter applications based on");
+ pw.println(" application state");
+ pw.println(" -status <Application ID> Prints the status of the application.");
+ pw.close();
+ String appsHelpStr = baos.toString("UTF-8");
+ return appsHelpStr;
+ }
+
+ private String createNodeCLIHelpMessage() throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("usage: node");
+ pw.println(" -all Works with -list to list all nodes.");
+ pw.println(" -list List all running nodes. Supports optional use of");
+ pw.println(" -states to filter nodes based on node state, all -all");
+ pw.println(" to list all nodes.");
+ pw.println(" -states <States> Works with -list to filter nodes based on input");
+ pw.println(" comma-separated list of node states.");
+ pw.println(" -status <NodeId> Prints the status report of the node.");
+ pw.close();
+ String nodesHelpStr = baos.toString("UTF-8");
+ return nodesHelpStr;
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java Tue Sep 3 23:20:45 2013
@@ -125,7 +125,7 @@ public class ResourceBlacklistRequestPBI
@Override
public void setBlacklistAdditions(List<String> resourceNames) {
- if (resourceNames == null) {
+ if (resourceNames == null || resourceNames.isEmpty()) {
if (this.blacklistAdditions != null) {
this.blacklistAdditions.clear();
}
@@ -144,7 +144,7 @@ public class ResourceBlacklistRequestPBI
@Override
public void setBlacklistRemovals(List<String> resourceNames) {
- if (resourceNames == null) {
+ if (resourceNames == null || resourceNames.isEmpty()) {
if (this.blacklistRemovals != null) {
this.blacklistRemovals.clear();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java Tue Sep 3 23:20:45 2013
@@ -39,6 +39,7 @@ public class ClientToAMTokenIdentifier e
public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN");
private ApplicationAttemptId applicationAttemptId;
+ private Text clientName = new Text();
// TODO: Add more information in the tokenID such that it is not
// transferrable, more secure etc.
@@ -46,21 +47,27 @@ public class ClientToAMTokenIdentifier e
public ClientToAMTokenIdentifier() {
}
- public ClientToAMTokenIdentifier(ApplicationAttemptId id) {
+ public ClientToAMTokenIdentifier(ApplicationAttemptId id, String client) {
this();
this.applicationAttemptId = id;
+ this.clientName = new Text(client);
}
public ApplicationAttemptId getApplicationAttemptID() {
return this.applicationAttemptId;
}
+ public String getClientName() {
+ return this.clientName.toString();
+ }
+
@Override
public void write(DataOutput out) throws IOException {
out.writeLong(this.applicationAttemptId.getApplicationId()
.getClusterTimestamp());
out.writeInt(this.applicationAttemptId.getApplicationId().getId());
out.writeInt(this.applicationAttemptId.getAttemptId());
+ this.clientName.write(out);
}
@Override
@@ -68,6 +75,7 @@ public class ClientToAMTokenIdentifier e
this.applicationAttemptId =
ApplicationAttemptId.newInstance(
ApplicationId.newInstance(in.readLong(), in.readInt()), in.readInt());
+ this.clientName.readFields(in);
}
@Override
@@ -77,10 +85,10 @@ public class ClientToAMTokenIdentifier e
@Override
public UserGroupInformation getUser() {
- if (this.applicationAttemptId == null) {
+ if (this.clientName == null) {
return null;
}
- return UserGroupInformation.createRemoteUser(this.applicationAttemptId.toString());
+ return UserGroupInformation.createRemoteUser(this.clientName.toString());
}
@InterfaceAudience.Private
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java Tue Sep 3 23:20:45 2013
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.record
public class ClientToAMTokenSecretManager extends
BaseClientToAMTokenSecretManager {
- // Only one client-token and one master-key for AM
+ // Only one master-key for AM
private SecretKey masterKey;
public ClientToAMTokenSecretManager(
@@ -53,7 +53,7 @@ public class ClientToAMTokenSecretManage
@Override
public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) {
- // Only one client-token and one master-key for AM, just return that.
+ // Only one master-key for AM, just return that.
return this.masterKey;
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Tue Sep 3 23:20:45 2013
@@ -198,7 +198,8 @@ public class ConverterUtils {
Iterator<String> it = _split(appIdStr).iterator();
if (!it.next().equals(APPLICATION_PREFIX)) {
throw new IllegalArgumentException("Invalid ApplicationId prefix: "
- + appIdStr);
+ + appIdStr + ". The valid ApplicationId should start with prefix "
+ + APPLICATION_PREFIX);
}
try {
return toApplicationId(it);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java Tue Sep 3 23:20:45 2013
@@ -242,9 +242,8 @@ public class WebApps {
for(Map.Entry<String, Object> entry : attributes.entrySet()) {
server.setAttribute(entry.getKey(), entry.getValue());
}
- String webAppPath = "/" + name + "/*";
server.defineFilter(server.getWebAppContext(), "guice",
- GuiceFilter.class.getName(), null, new String[] { webAppPath, "/" });
+ GuiceFilter.class.getName(), null, new String[] { "/*" });
webapp.setConf(conf);
webapp.setHttpServer(server);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java Tue Sep 3 23:20:45 2013
@@ -19,11 +19,14 @@
package org.apache.hadoop.yarn.util;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.BreakableService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@@ -314,6 +317,26 @@ public class TestCompositeService {
composite.init(new Configuration());
assertInState(STATE.INITED, child);
}
+
+ @Test (timeout = 1000)
+ public void testAddIfService() {
+ CompositeService testService = new CompositeService("TestService") {
+ Service service;
+ @Override
+ public void serviceInit(Configuration conf) {
+ Integer notAService = new Integer(0);
+ assertFalse("Added an integer as a service",
+ addIfService(notAService));
+
+ service = new AbstractService("Service") {};
+ assertTrue("Unable to add a service", addIfService(service));
+ }
+ };
+
+ testService.init(new Configuration());
+ assertEquals("Incorrect number of services",
+ 1, testService.getServices().size());
+ }
public static class CompositeServiceAddingAChild extends CompositeService{
Service child;
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java Tue Sep 3 23:20:45 2013
@@ -18,30 +18,47 @@
package org.apache.hadoop.yarn.webapp;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebApps;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage;
-import org.apache.hadoop.yarn.webapp.view.JQueryUI;
-import org.apache.hadoop.yarn.webapp.view.TextPage;
-
-import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import javax.xml.bind.JAXBContext;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
-
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.webapp.view.HtmlPage;
+import org.apache.hadoop.yarn.webapp.view.JQueryUI;
+import org.apache.hadoop.yarn.webapp.view.TextPage;
import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+
public class TestWebApp {
static final Logger LOG = LoggerFactory.getLogger(TestWebApp.class);
@@ -227,14 +244,19 @@ public class TestWebApp {
}
@Test public void testCustomRoutes() throws Exception {
- WebApp app = WebApps.$for("test", this).start(new WebApp() {
- @Override public void setup() {
- route("/:foo", FooController.class);
- route("/bar/foo", FooController.class, "bar");
- route("/foo/:foo", DefaultController.class);
- route("/foo/bar/:foo", DefaultController.class, "index");
- }
- });
+ WebApp app =
+ WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
+ @Override
+ public void setup() {
+ bind(MyTestJAXBContextResolver.class);
+ bind(MyTestWebService.class);
+
+ route("/:foo", FooController.class);
+ route("/bar/foo", FooController.class, "bar");
+ route("/foo/:foo", DefaultController.class);
+ route("/foo/bar/:foo", DefaultController.class, "index");
+ }
+ });
String baseUrl = baseUrl(app);
try {
assertEquals("foo", getContent(baseUrl).trim());
@@ -245,6 +267,31 @@ public class TestWebApp {
assertEquals("default1", getContent(baseUrl +"test/foo/1").trim());
assertEquals("default2", getContent(baseUrl +"test/foo/bar/2").trim());
assertEquals(404, getResponseCode(baseUrl +"test/goo"));
+ assertEquals(200, getResponseCode(baseUrl +"ws/v1/test"));
+ assertTrue(getContent(baseUrl +"ws/v1/test").contains("myInfo"));
+ } finally {
+ app.stop();
+ }
+ }
+
+ // This is to test the GuiceFilter should only be applied to webAppContext,
+ // not to staticContext and logContext;
+ @Test public void testYARNWebAppContext() throws Exception {
+ // setting up the log context
+ System.setProperty("hadoop.log.dir", "/Not/Existing/dir");
+ WebApp app = WebApps.$for("test", this).start(new WebApp() {
+ @Override public void setup() {
+ route("/", FooController.class);
+ }
+ });
+ String baseUrl = baseUrl(app);
+ try {
+ // should not redirect to foo
+ assertFalse("foo".equals(getContent(baseUrl +"static").trim()));
+ // Not able to access a non-existing dir, should not redirect to foo.
+ assertEquals(404, getResponseCode(baseUrl +"logs"));
+ // should be able to redirect to foo.
+ assertEquals("foo", getContent(baseUrl).trim());
} finally {
app.stop();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Tue Sep 3 23:20:45 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
/**
* Context interface for sharing information across components in the
@@ -61,4 +62,8 @@ public interface Context {
NodeHealthStatus getNodeHealthStatus();
ContainerManagementProtocol getContainerManager();
+
+ LocalDirsHandlerService getLocalDirsHandler();
+
+ ApplicationACLsManager getApplicationACLsManager();
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Sep 3 23:20:45 2013
@@ -123,7 +123,8 @@ public class NodeManager extends Composi
protected NMContext createNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager) {
- return new NMContext(containerTokenSecretManager, nmTokenSecretManager);
+ return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
+ dirsHandler, aclsManager);
}
protected void doSecureLogin() throws IOException {
@@ -142,9 +143,6 @@ public class NodeManager extends Composi
NMTokenSecretManagerInNM nmTokenSecretManager =
new NMTokenSecretManagerInNM();
- this.context =
- createNMContext(containerTokenSecretManager, nmTokenSecretManager);
-
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.newInstance(
@@ -165,7 +163,9 @@ public class NodeManager extends Composi
addService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.getDiskHandler();
-
+ this.context = createNMContext(containerTokenSecretManager,
+ nmTokenSecretManager);
+
nodeStatusUpdater =
createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
@@ -319,14 +319,19 @@ public class NodeManager extends Composi
private final NMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInNM nmTokenSecretManager;
private ContainerManagementProtocol containerManager;
+ private final LocalDirsHandlerService dirsHandler;
+ private final ApplicationACLsManager aclsManager;
private WebServer webServer;
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
-
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
- NMTokenSecretManagerInNM nmTokenSecretManager) {
+ NMTokenSecretManagerInNM nmTokenSecretManager,
+ LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager) {
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
+ this.dirsHandler = dirsHandler;
+ this.aclsManager = aclsManager;
this.nodeHealthStatus.setIsNodeHealthy(true);
this.nodeHealthStatus.setHealthReport("Healthy");
this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
@@ -386,6 +391,16 @@ public class NodeManager extends Composi
public void setNodeId(NodeId nodeId) {
this.nodeId = nodeId;
}
+
+ @Override
+ public LocalDirsHandlerService getLocalDirsHandler() {
+ return dirsHandler;
+ }
+
+ @Override
+ public ApplicationACLsManager getApplicationACLsManager() {
+ return aclsManager;
+ }
}
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Sep 3 23:20:45 2013
@@ -85,7 +85,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -193,12 +192,6 @@ public class ContainerManagerImpl extend
super.serviceInit(conf);
}
- private void addIfService(Object object) {
- if (object instanceof Service) {
- addService((Service) object);
- }
- }
-
protected LogHandler createLogHandler(Configuration conf, Context context,
DeletionService deletionService) {
if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
@@ -475,8 +468,7 @@ public class ContainerManagerImpl extend
// Create the application
Application application =
- new ApplicationImpl(dispatcher, this.aclsManager, user, applicationID,
- credentials, context);
+ new ApplicationImpl(dispatcher, user, applicationID, credentials, context);
if (null == context.getApplications().putIfAbsent(applicationID,
application)) {
LOG.info("Creating a new application reference for app " + applicationID);
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Sep 3 23:20:45 2013
@@ -73,14 +73,13 @@ public class ApplicationImpl implements
Map<ContainerId, Container> containers =
new HashMap<ContainerId, Container>();
- public ApplicationImpl(Dispatcher dispatcher,
- ApplicationACLsManager aclsManager, String user, ApplicationId appId,
+ public ApplicationImpl(Dispatcher dispatcher, String user, ApplicationId appId,
Credentials credentials, Context context) {
this.dispatcher = dispatcher;
this.user = user;
this.appId = appId;
this.credentials = credentials;
- this.aclsManager = aclsManager;
+ this.aclsManager = context.getApplicationACLsManager();
this.context = context;
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Tue Sep 3 23:20:45 2013
@@ -525,7 +525,8 @@ public class ContainerLaunch implements
@Override
public void env(String key, String value) {
- line("@set ", key, "=", value);
+ line("@set ", key, "=", value,
+ "\nif %errorlevel% neq 0 exit /b %errorlevel%");
}
@Override
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1519885&r1=1519884&r2=1519885&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Tue Sep 3 23:20:45 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -149,7 +150,8 @@ public class ContainersLauncher extends
dispatcher.getEventHandler().handle(
new ContainerExitEvent(containerId,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
- ExitCode.TERMINATED.getExitCode(),
+ Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
+ ExitCode.TERMINATED.getExitCode(),
"Container terminated before launch."));
}
}