You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/24 04:22:00 UTC
svn commit: r1471229 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java...
Author: szetszwo
Date: Wed Apr 24 02:21:58 2013
New Revision: 1471229
URL: http://svn.apache.org/r1471229
Log:
Merge r1470760 through r1471228 from trunk.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Wed Apr 24 02:21:58 2013
@@ -95,6 +95,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-441. Removed unused utility methods for collections from two API
records. (Xuan Gong via vinodkv)
+ YARN-561. Modified NodeManager to set key information into the environment
+ of every container that it launches. (Xuan Gong via vinodkv)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
@@ -167,6 +170,13 @@ Release 2.0.5-beta - UNRELEASED
YARN-542. Changed the default global AM max-attempts value to be not one.
(Zhijie Shen via vinodkv)
+ YARN-583. Moved application level local resources to be localized under the
+ filecache sub-directory under application directory. (Omkar Vinit Joshi via
+ vinodkv)
+
+ YARN-581. Added a test to verify that app delegation tokens are restored
+ after RM restart. (Jian He via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -277,6 +287,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-594. Update test and add comments in YARN-534 (Jian He via bikas)
+ YARN-549. YarnClient.submitApplication should wait for application to be
+ accepted by the RM (Zhijie Shen via bikas)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Wed Apr 24 02:21:58 2013
@@ -38,30 +38,6 @@ public interface ApplicationConstants {
"AppClientSecretEnv";
/**
- * The environment variable for CONTAINER_ID. Set in AppMaster environment
- * only
- */
- public static final String AM_CONTAINER_ID_ENV = "AM_CONTAINER_ID";
-
- /**
- * The environment variable for the NM_HOST. Set in the AppMaster environment
- * only
- */
- public static final String NM_HOST_ENV = "NM_HOST";
-
- /**
- * The environment variable for the NM_PORT. Set in the AppMaster environment
- * only
- */
- public static final String NM_PORT_ENV = "NM_PORT";
-
- /**
- * The environment variable for the NM_HTTP_PORT. Set in the AppMaster environment
- * only
- */
- public static final String NM_HTTP_PORT_ENV = "NM_HTTP_PORT";
-
- /**
* The environment variable for APP_SUBMIT_TIME. Set in AppMaster environment
* only
*/
@@ -70,8 +46,6 @@ public interface ApplicationConstants {
public static final String CONTAINER_TOKEN_FILE_ENV_NAME =
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
- public static final String LOCAL_DIR_ENV = "YARN_LOCAL_DIRS";
-
/**
* The environmental variable for APPLICATION_WEB_PROXY_BASE. Set in
* ApplicationMaster's environment only. This states that for all non-relative
@@ -177,7 +151,37 @@ public interface ApplicationConstants {
/**
* $HADOOP_YARN_HOME
*/
- HADOOP_YARN_HOME("HADOOP_YARN_HOME");
+ HADOOP_YARN_HOME("HADOOP_YARN_HOME"),
+
+ /**
+ * $CONTAINER_ID
+ * Final, exported by NodeManager and non-modifiable by users.
+ */
+ CONTAINER_ID("CONTAINER_ID"),
+
+ /**
+ * $NM_HOST
+ * Final, exported by NodeManager and non-modifiable by users.
+ */
+ NM_HOST("NM_HOST"),
+
+ /**
+ * $NM_HTTP_PORT
+ * Final, exported by NodeManager and non-modifiable by users.
+ */
+ NM_HTTP_PORT("NM_HTTP_PORT"),
+
+ /**
+ * $NM_PORT
+ * Final, exported by NodeManager and non-modifiable by users.
+ */
+ NM_PORT("NM_PORT"),
+
+ /**
+ * $LOCAL_DIRS
+ * Final, exported by NodeManager and non-modifiable by users.
+ */
+ LOCAL_DIRS("LOCAL_DIRS");
private final String variable;
private Environment(String variable) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ClientRMProtocol.java Wed Apr 24 02:21:58 2013
@@ -96,7 +96,9 @@ public interface ClientRMProtocol {
*
* <p>Currently the <code>ResourceManager</code> sends an immediate (empty)
* {@link SubmitApplicationResponse} on accepting the submission and throws
- * an exception if it rejects the submission.</p>
+ * an exception if it rejects the submission. However, this call needs to be
+ * followed by {@link #getApplicationReport(GetApplicationReportRequest)}
+ * to make sure that the application gets properly submitted.</p>
*
* <p> In secure mode,the <code>ResourceManager</code> verifies access to
* queues etc. before accepting the application submission.</p>
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Wed Apr 24 02:21:58 2013
@@ -45,6 +45,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.ContainerExitStatus;
import org.apache.hadoop.yarn.api.ContainerManager;
@@ -320,7 +321,7 @@ public class ApplicationMaster {
Map<String, String> envs = System.getenv();
- if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
+ if (!envs.containsKey(Environment.CONTAINER_ID.name())) {
if (cliParser.hasOption("app_attempt_id")) {
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
@@ -330,7 +331,7 @@ public class ApplicationMaster {
}
} else {
ContainerId containerId = ConverterUtils.toContainerId(envs
- .get(ApplicationConstants.AM_CONTAINER_ID_ENV));
+ .get(Environment.CONTAINER_ID.name()));
appAttemptID = containerId.getApplicationAttemptId();
}
@@ -338,16 +339,16 @@ public class ApplicationMaster {
throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV
+ " not set in the environment");
}
- if (!envs.containsKey(ApplicationConstants.NM_HOST_ENV)) {
- throw new RuntimeException(ApplicationConstants.NM_HOST_ENV
+ if (!envs.containsKey(Environment.NM_HOST.name())) {
+ throw new RuntimeException(Environment.NM_HOST.name()
+ " not set in the environment");
}
- if (!envs.containsKey(ApplicationConstants.NM_HTTP_PORT_ENV)) {
- throw new RuntimeException(ApplicationConstants.NM_HTTP_PORT_ENV
+ if (!envs.containsKey(Environment.NM_HTTP_PORT.name())) {
+ throw new RuntimeException(Environment.NM_HTTP_PORT
+ " not set in the environment");
}
- if (!envs.containsKey(ApplicationConstants.NM_PORT_ENV)) {
- throw new RuntimeException(ApplicationConstants.NM_PORT_ENV
+ if (!envs.containsKey(Environment.NM_PORT.name())) {
+ throw new RuntimeException(Environment.NM_PORT.name()
+ " not set in the environment");
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/src/main/java/org/apache/hadoop/yarn/applications/unmanagedamlauncher/UnmanagedAMLauncher.java Wed Apr 24 02:21:58 2013
@@ -37,6 +37,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -58,7 +59,7 @@ import org.apache.hadoop.yarn.util.Recor
* creates a new application on the RM and negotiates a new attempt id. Then it
* waits for the RM app state to reach be YarnApplicationState.ACCEPTED after
* which it spawns the AM in another process and passes it the container id via
- * env variable ApplicationConstants.AM_CONTAINER_ID_ENV. The AM can be in any
+ * env variable Environment.CONTAINER_ID. The AM can be in any
* language. The AM can register with the RM using the attempt id obtained
* from the container id and proceed as normal.
* The client redirects app stdout and stderr to its own stdout and
@@ -190,10 +191,11 @@ public class UnmanagedAMLauncher {
containerId.setId(0);
String hostname = InetAddress.getLocalHost().getHostName();
- envAMList.add(ApplicationConstants.AM_CONTAINER_ID_ENV + "=" + containerId);
- envAMList.add(ApplicationConstants.NM_HOST_ENV + "=" + hostname);
- envAMList.add(ApplicationConstants.NM_HTTP_PORT_ENV + "=0");
- envAMList.add(ApplicationConstants.NM_PORT_ENV + "=0");
+ envAMList.add(Environment.CONTAINER_ID.name() + "=" + containerId);
+ envAMList.add(Environment.NM_HOST.name() + "=" + hostname);
+ envAMList.add(Environment.NM_HTTP_PORT.name() + "=0");
+ envAMList.add(Environment.NM_PORT.name() + "=0");
+ envAMList.add(Environment.LOCAL_DIRS.name() + "= /tmp");
envAMList.add(ApplicationConstants.APP_SUBMIT_TIME_ENV + "="
+ System.currentTimeMillis());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClient.java Wed Apr 24 02:21:58 2013
@@ -63,7 +63,9 @@ public interface YarnClient extends Serv
/**
* <p>
- * Submit a new application to <code>YARN.</code>
+ * Submit a new application to <code>YARN.</code> It is a blocking call, such
+ * that it will not return {@link ApplicationId} until the submitted
+ * application has been submitted and accepted by the ResourceManager.
* </p>
*
* @param appContext
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/YarnClientImpl.java Wed Apr 24 02:21:58 2013
@@ -53,9 +53,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.Records;
@@ -68,6 +70,7 @@ public class YarnClientImpl extends Abst
protected ClientRMProtocol rmClient;
protected InetSocketAddress rmAddress;
+ protected long statePollIntervalMillis;
private static final String ROOT = "root";
@@ -90,6 +93,9 @@ public class YarnClientImpl extends Abst
if (this.rmAddress == null) {
this.rmAddress = getRmAddress(conf);
}
+ statePollIntervalMillis = conf.getLong(
+ YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS);
super.init(conf);
}
@@ -131,6 +137,29 @@ public class YarnClientImpl extends Abst
Records.newRecord(SubmitApplicationRequest.class);
request.setApplicationSubmissionContext(appContext);
rmClient.submitApplication(request);
+
+ int pollCount = 0;
+ while (true) {
+ YarnApplicationState state =
+ getApplicationReport(applicationId).getYarnApplicationState();
+ if (!state.equals(YarnApplicationState.NEW) &&
+ !state.equals(YarnApplicationState.NEW_SAVING)) {
+ break;
+ }
+ // Notify the client through the log every 10 poll, in case the client
+ // is blocked here too long.
+ if (++pollCount % 10 == 0) {
+ LOG.info("Application submission is not finished, " +
+ "submitted application " + applicationId +
+ " is still in " + state);
+ }
+ try {
+ Thread.sleep(statePollIntervalMillis);
+ } catch (InterruptedException ie) {
+ }
+ }
+
+
LOG.info("Submitted application " + applicationId + " to ResourceManager"
+ " at " + rmAddress);
return applicationId;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestYarnClient.java Wed Apr 24 02:21:58 2013
@@ -18,10 +18,25 @@
package org.apache.hadoop.yarn.client;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import junit.framework.Assert;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.client.YarnClient;
-import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.api.ClientRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
public class TestYarnClient {
@@ -43,4 +58,76 @@ public class TestYarnClient {
client.start();
client.stop();
}
+
+ @Test (timeout = 30000)
+ public void testSubmitApplication() {
+ Configuration conf = new Configuration();
+ conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
+ 100); // speed up tests
+ final YarnClient client = new MockYarnClient();
+ client.init(conf);
+ client.start();
+
+ YarnApplicationState[] exitStates = new YarnApplicationState[]
+ {
+ YarnApplicationState.SUBMITTED,
+ YarnApplicationState.ACCEPTED,
+ YarnApplicationState.RUNNING,
+ YarnApplicationState.FINISHED,
+ YarnApplicationState.FAILED,
+ YarnApplicationState.KILLED
+ };
+ for (int i = 0; i < exitStates.length; ++i) {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ApplicationId applicationId = Records.newRecord(ApplicationId.class);
+ applicationId.setClusterTimestamp(System.currentTimeMillis());
+ applicationId.setId(i);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ ((MockYarnClient) client).setYarnApplicationState(exitStates[i]);
+ try {
+ client.submitApplication(context);
+ } catch (YarnRemoteException e) {
+ Assert.fail("Exception is not expected.");
+ }
+ verify(((MockYarnClient) client).mockReport,times(4 * i + 4))
+ .getYarnApplicationState();
+ }
+
+ client.stop();
+ }
+
+ private static class MockYarnClient extends YarnClientImpl {
+ private ApplicationReport mockReport;
+
+ public MockYarnClient() {
+ super();
+ }
+
+ @Override
+ public void start() {
+ rmClient = mock(ClientRMProtocol.class);
+ GetApplicationReportResponse mockResponse =
+ mock(GetApplicationReportResponse.class);
+ mockReport = mock(ApplicationReport.class);
+ try{
+ when(rmClient.getApplicationReport(any(
+ GetApplicationReportRequest.class))).thenReturn(mockResponse);
+ } catch (YarnRemoteException e) {
+ Assert.fail("Exception is not expected.");
+ }
+ when(mockResponse.getApplicationReport()).thenReturn(mockReport);
+ }
+
+ @Override
+ public void stop() {
+ }
+
+ public void setYarnApplicationState(YarnApplicationState state) {
+ when(mockReport.getYarnApplicationState()).thenReturn(
+ YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING,
+ YarnApplicationState.NEW_SAVING, state);
+ }
+ }
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Apr 24 02:21:58 2013
@@ -692,6 +692,19 @@ public class YarnConfiguration extends C
*/
public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
+ ////////////////////////////////
+ // Other Configs
+ ////////////////////////////////
+
+ /**
+ * The interval of the yarn client's querying application state after
+ * application submission. The unit is millisecond.
+ */
+ public static final String YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
+ YARN_PREFIX + "client.app-submission.poll-interval";
+ public static final long DEFAULT_YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS =
+ 1000;
+
public YarnConfiguration() {
super();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Wed Apr 24 02:21:58 2013
@@ -708,4 +708,12 @@
<value>$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/share/hadoop/common/*,$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,$HADOOP_YARN_HOME/share/hadoop/yarn/*,$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*</value>
</property>
+ <!-- Other configuration -->
+ <property>
+ <description>The interval of the yarn client's querying application state
+ after application submission. The unit is millisecond.</description>
+ <name>yarn.client.app-submission.poll-interval</name>
+ <value>1000</value>
+ </property>
+
</configuration>
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Wed Apr 24 02:21:58 2013
@@ -113,13 +113,13 @@ public class DefaultContainerExecutor ex
List<String> localDirs, List<String> logDirs) throws IOException {
FsPermission dirPerm = new FsPermission(APPDIR_PERM);
- ContainerId containerId = container.getContainerID();
+ ContainerId containerId = container.getContainer().getId();
// create container dirs on all disks
String containerIdStr = ConverterUtils.toString(containerId);
String appIdStr =
ConverterUtils.toString(
- container.getContainerID().getApplicationAttemptId().
+ containerId.getApplicationAttemptId().
getApplicationId());
for (String sLocalDir : localDirs) {
Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Wed Apr 24 02:21:58 2013
@@ -216,11 +216,11 @@ public class LinuxContainerExecutor exte
String user, String appId, Path containerWorkDir,
List<String> localDirs, List<String> logDirs) throws IOException {
- ContainerId containerId = container.getContainerID();
+ ContainerId containerId = container.getContainer().getId();
String containerIdStr = ConverterUtils.toString(containerId);
resourcesHandler.preExecute(containerId,
- container.getResource());
+ container.getContainer().getResource());
String resourcesOptions = resourcesHandler.getResourcesOption(
containerId);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java Wed Apr 24 02:21:58 2013
@@ -36,8 +36,8 @@ public class ApplicationContainerInitEve
final Container container;
public ApplicationContainerInitEvent(Container container) {
- super(container.getContainerID().getApplicationAttemptId().getApplicationId(),
- ApplicationEventType.INIT_CONTAINER);
+ super(container.getContainer().getId().getApplicationAttemptId()
+ .getApplicationId(), ApplicationEventType.INIT_CONTAINER);
this.container = container;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Wed Apr 24 02:21:58 2013
@@ -274,14 +274,14 @@ public class ApplicationImpl implements
ApplicationContainerInitEvent initEvent =
(ApplicationContainerInitEvent) event;
Container container = initEvent.getContainer();
- app.containers.put(container.getContainerID(), container);
- LOG.info("Adding " + container.getContainerID()
+ app.containers.put(container.getContainer().getId(), container);
+ LOG.info("Adding " + container.getContainer().getId()
+ " to application " + app.toString());
switch (app.getApplicationState()) {
case RUNNING:
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainerID()));
+ container.getContainer().getId()));
break;
case INITING:
case NEW:
@@ -302,7 +302,7 @@ public class ApplicationImpl implements
// Start all the containers waiting for ApplicationInit
for (Container container : app.containers.values()) {
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainerID()));
+ container.getContainer().getId()));
}
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java Wed Apr 24 02:21:58 2013
@@ -25,12 +25,11 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
public interface Container extends EventHandler<ContainerEvent> {
- org.apache.hadoop.yarn.api.records.ContainerId getContainerID();
+ org.apache.hadoop.yarn.api.records.Container getContainer();
String getUser();
@@ -46,5 +45,4 @@ public interface Container extends Event
String toString();
- Resource getResource();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Wed Apr 24 02:21:58 2013
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -313,16 +312,6 @@ public class ContainerImpl implements Co
}
@Override
- public ContainerId getContainerID() {
- this.readLock.lock();
- try {
- return this.container.getId();
- } finally {
- this.readLock.unlock();
- }
- }
-
- @Override
public String getUser() {
this.readLock.lock();
try {
@@ -385,10 +374,10 @@ public class ContainerImpl implements Co
}
@Override
- public Resource getResource() {
+ public org.apache.hadoop.yarn.api.records.Container getContainer() {
this.readLock.lock();
try {
- return this.container.getResource();
+ return this.container;
} finally {
this.readLock.unlock();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Wed Apr 24 02:21:58 2013
@@ -118,7 +118,7 @@ public class ContainerLaunch implements
final ContainerLaunchContext launchContext = container.getLaunchContext();
final Map<Path,List<String>> localResources =
container.getLocalizedResources();
- ContainerId containerID = container.getContainerID();
+ ContainerId containerID = container.getContainer().getId();
String containerIdStr = ConverterUtils.toString(containerID);
final String user = launchContext.getUser();
final List<String> command = launchContext.getCommands();
@@ -299,7 +299,7 @@ public class ContainerLaunch implements
* @throws IOException
*/
public void cleanupContainer() throws IOException {
- ContainerId containerId = container.getContainerID();
+ ContainerId containerId = container.getContainer().getId();
String containerIdStr = ConverterUtils.toString(containerId);
LOG.info("Cleaning up container " + containerIdStr);
@@ -370,7 +370,7 @@ public class ContainerLaunch implements
*/
private String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
- ConverterUtils.toString(container.getContainerID());
+ ConverterUtils.toString(container.getContainer().getId());
String processId = null;
LOG.debug("Accessing pid for container " + containerIdStr
+ " from pid file " + pidFilePath);
@@ -547,6 +547,21 @@ public class ContainerLaunch implements
* Non-modifiable environment variables
*/
+ environment.put(Environment.CONTAINER_ID.name(), container
+ .getContainer().getId().toString());
+
+ environment.put(Environment.NM_PORT.name(),
+ String.valueOf(container.getContainer().getNodeId().getPort()));
+
+ environment.put(Environment.NM_HOST.name(), container.getContainer()
+ .getNodeId().getHost());
+
+ environment.put(Environment.NM_HTTP_PORT.name(), container.getContainer()
+ .getNodeHttpAddress().split(":")[1]);
+
+ environment.put(Environment.LOCAL_DIRS.name(),
+ StringUtils.join(",", appDirs));
+
putEnvIfNotNull(environment, Environment.USER.name(), container.getUser());
putEnvIfNotNull(environment,
@@ -566,11 +581,6 @@ public class ContainerLaunch implements
Environment.HADOOP_CONF_DIR.name(),
System.getenv(Environment.HADOOP_CONF_DIR.name())
);
-
- putEnvIfNotNull(environment,
- ApplicationConstants.LOCAL_DIR_ENV,
- StringUtils.join(",", appDirs)
- );
if (!Shell.WINDOWS) {
environment.put("JVM_PID", "$$");
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Wed Apr 24 02:21:58 2013
@@ -111,7 +111,7 @@ public class ContainersLauncher extends
public void handle(ContainersLauncherEvent event) {
// TODO: ContainersLauncher launches containers one by one!!
Container container = event.getContainer();
- ContainerId containerId = container.getContainerID();
+ ContainerId containerId = container.getContainer().getId();
switch (event.getType()) {
case LAUNCH_CONTAINER:
Application app =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Wed Apr 24 02:21:58 2013
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
@@ -46,7 +47,6 @@ import java.util.concurrent.ScheduledExe
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -65,6 +65,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.DiskChecker;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -358,13 +359,15 @@ public class ResourceLocalizationService
ContainerLocalizationRequestEvent rsrcReqs) {
Container c = rsrcReqs.getContainer();
LocalizerContext ctxt = new LocalizerContext(
- c.getUser(), c.getContainerID(), c.getCredentials());
+ c.getUser(), c.getContainer().getId(), c.getCredentials());
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
rsrcReqs.getRequestedResources();
for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
rsrcs.entrySet()) {
- LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
- c.getContainerID().getApplicationAttemptId().getApplicationId());
+ LocalResourcesTracker tracker =
+ getLocalResourcesTracker(e.getKey(), c.getUser(),
+ c.getContainer().getId().getApplicationAttemptId()
+ .getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
}
@@ -393,19 +396,21 @@ public class ResourceLocalizationService
for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
rsrcs.entrySet()) {
LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
- c.getContainerID().getApplicationAttemptId().getApplicationId());
+ c.getContainer().getId().getApplicationAttemptId()
+ .getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
- tracker.handle(new ResourceReleaseEvent(req, c.getContainerID()));
+ tracker.handle(new ResourceReleaseEvent(req,
+ c.getContainer().getId()));
}
}
- String locId = ConverterUtils.toString(c.getContainerID());
+ String locId = ConverterUtils.toString(c.getContainer().getId());
localizerTracker.cleanupPrivLocalizers(locId);
// Delete the container directories
String userName = c.getUser();
String containerIDStr = c.toString();
String appIDStr = ConverterUtils.toString(
- c.getContainerID().getApplicationAttemptId().getApplicationId());
+ c.getContainer().getId().getApplicationAttemptId().getApplicationId());
for (String localDir : dirsHandler.getLocalDirs()) {
// Delete the user-owned container-dir
@@ -424,8 +429,9 @@ public class ResourceLocalizationService
delService.delete(null, containerSysDir, new Path[] {});
}
- dispatcher.getEventHandler().handle(new ContainerEvent(c.getContainerID(),
- ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
+ dispatcher.getEventHandler().handle(
+ new ContainerEvent(c.getContainer().getId(),
+ ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
}
@@ -481,18 +487,15 @@ public class ResourceLocalizationService
}
private String getUserFileCachePath(String user) {
- String path =
- "." + Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR
- + user + Path.SEPARATOR + ContainerLocalizer.FILECACHE;
- return path;
+ return StringUtils.join(Path.SEPARATOR, Arrays.asList(".",
+ ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));
+
}
- private String getUserAppCachePath(String user, String appId) {
- String path =
- "." + Path.SEPARATOR + ContainerLocalizer.USERCACHE + Path.SEPARATOR
- + user + Path.SEPARATOR + ContainerLocalizer.APPCACHE
- + Path.SEPARATOR + appId;
- return path;
+ private String getAppFileCachePath(String user, String appId) {
+ return StringUtils.join(Path.SEPARATOR, Arrays.asList(".",
+ ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId,
+ ContainerLocalizer.FILECACHE));
}
@VisibleForTesting
@@ -942,7 +945,7 @@ public class ResourceLocalizationService
if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only
cacheDirectory = getUserFileCachePath(user);
} else {// APPLICATION ONLY
- cacheDirectory = getUserAppCachePath(user, appId.toString());
+ cacheDirectory = getAppFileCachePath(user, appId.toString());
}
Path dirPath =
dirsHandler.getLocalPathForWrite(cacheDirectory,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java Wed Apr 24 02:21:58 2013
@@ -60,7 +60,7 @@ public class ContainerInfo {
public ContainerInfo(final Context nmContext, final Container container,
String requestUri, String pathPrefix) {
- this.id = container.getContainerID().toString();
+ this.id = container.getContainer().getId().toString();
this.nodeId = nmContext.getNodeId().toString();
ContainerStatus containerData = container.cloneAndGetContainerStatus();
this.exitCode = containerData.getExitStatus();
@@ -74,7 +74,7 @@ public class ContainerInfo {
}
this.user = container.getUser();
- Resource res = container.getResource();
+ Resource res = container.getContainer().getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Wed Apr 24 02:21:58 2013
@@ -88,10 +88,10 @@ public class DummyContainerManager exten
.getRequestedResources().values()) {
for (LocalResourceRequest req : rc) {
LOG.info("DEBUG: " + req + ":"
- + rsrcReqs.getContainer().getContainerID());
+ + rsrcReqs.getContainer().getContainer().getId());
dispatcher.getEventHandler().handle(
new ContainerResourceLocalizedEvent(rsrcReqs.getContainer()
- .getContainerID(), req, new Path("file:///local"
+ .getContainer().getId(), req, new Path("file:///local"
+ req.getPath().toUri().getPath())));
}
}
@@ -101,7 +101,7 @@ public class DummyContainerManager exten
((ContainerLocalizationEvent) event).getContainer();
// TODO: delete the container dir
this.dispatcher.getEventHandler().handle(
- new ContainerEvent(container.getContainerID(),
+ new ContainerEvent(container.getContainer().getId(),
ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
break;
case DESTROY_APPLICATION_RESOURCES:
@@ -130,7 +130,7 @@ public class DummyContainerManager exten
@Override
public void handle(ContainersLauncherEvent event) {
Container container = event.getContainer();
- ContainerId containerId = container.getContainerID();
+ ContainerId containerId = container.getContainer().getId();
switch (event.getType()) {
case LAUNCH_CONTAINER:
dispatcher.getEventHandler().handle(
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java Wed Apr 24 02:21:58 2013
@@ -186,7 +186,10 @@ public class TestLinuxContainerExecutor
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap<String, String> env = new HashMap<String,String>();
- when(container.getContainerID()).thenReturn(cId);
+ org.apache.hadoop.yarn.api.records.Container containerAPI =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(container.getContainer()).thenReturn(containerAPI);
+ when(container.getContainer().getId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(context.getEnvironment()).thenReturn(env);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Wed Apr 24 02:21:58 2013
@@ -107,7 +107,10 @@ public class TestLinuxContainerExecutorW
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap<String, String> env = new HashMap<String,String>();
- when(container.getContainerID()).thenReturn(cId);
+ org.apache.hadoop.yarn.api.records.Container containerAPI =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(container.getContainer()).thenReturn(containerAPI);
+ when(container.getContainer().getId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(cId.toString()).thenReturn(containerId);
@@ -225,7 +228,10 @@ public class TestLinuxContainerExecutorW
ContainerLaunchContext context = mock(ContainerLaunchContext.class);
HashMap<String, String> env = new HashMap<String, String>();
- when(container.getContainerID()).thenReturn(cId);
+ org.apache.hadoop.yarn.api.records.Container containerAPI =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(container.getContainer()).thenReturn(containerAPI);
+ when(container.getContainer().getId()).thenReturn(cId);
when(container.getLaunchContext()).thenReturn(context);
when(cId.toString()).thenReturn(containerId);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Wed Apr 24 02:21:58 2013
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -178,6 +179,9 @@ public class TestNodeManagerShutdown {
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
+ NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234);
+ when(mockContainer.getNodeId()).thenReturn(nodeId);
+ when(mockContainer.getNodeHttpAddress()).thenReturn("localhost:12345");
containerLaunchContext.setUser(user);
URL localResourceUri =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Wed Apr 24 02:21:58 2013
@@ -150,6 +150,10 @@ public class TestContainerManager extend
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(512, 1));
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeHttpAddress()).thenReturn(
+ context.getNodeId().getHost() + ":12345");
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
@@ -245,6 +249,10 @@ public class TestContainerManager extend
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1)); // MB
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeHttpAddress()).thenReturn(
+ context.getNodeId().getHost() + ":12345");
+
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
@@ -352,7 +360,9 @@ public class TestContainerManager extend
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1)); // MB
-
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeHttpAddress()).thenReturn(
+ context.getNodeId().getHost() + ":12345");
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
@@ -444,6 +454,9 @@ public class TestContainerManager extend
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
BuilderUtils.newResource(100, 1));
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeHttpAddress()).thenReturn(
+ context.getNodeId().getHost() + ":12345");
// containerLaunchContext.command = new ArrayList<CharSequence>();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Wed Apr 24 02:21:58 2013
@@ -83,7 +83,7 @@ public class TestApplication {
for (int i = 0; i < wa.containers.size(); i++) {
verify(wa.containerBus).handle(
argThat(new ContainerInitMatcher(wa.containers.get(i)
- .getContainerID())));
+ .getContainer().getId())));
}
} finally {
if (wa != null)
@@ -108,7 +108,7 @@ public class TestApplication {
assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
verify(wa.containerBus).handle(
argThat(new ContainerInitMatcher(wa.containers.get(0)
- .getContainerID())));
+ .getContainer().getId())));
wa.initContainer(1);
wa.initContainer(2);
@@ -118,7 +118,7 @@ public class TestApplication {
for (int i = 1; i < wa.containers.size(); i++) {
verify(wa.containerBus).handle(
argThat(new ContainerInitMatcher(wa.containers.get(i)
- .getContainerID())));
+ .getContainer().getId())));
}
} finally {
if (wa != null)
@@ -233,7 +233,7 @@ public class TestApplication {
for (int i = 1; i < wa.containers.size(); i++) {
verify(wa.containerBus).handle(
argThat(new ContainerKillMatcher(wa.containers.get(i)
- .getContainerID())));
+ .getContainer().getId())));
}
wa.containerFinished(1);
@@ -354,7 +354,7 @@ public class TestApplication {
verify(wa.containerBus).handle(
argThat(new ContainerKillMatcher(wa.containers.get(0)
- .getContainerID())));
+ .getContainer().getId())));
assertEquals(ApplicationState.FINISHING_CONTAINERS_WAIT,
wa.app.getApplicationState());
@@ -487,7 +487,7 @@ public class TestApplication {
public void containerFinished(int containerNum) {
app.handle(new ApplicationContainerFinishedEvent(containers.get(
- containerNum).getContainerID()));
+ containerNum).getContainer().getId()));
drainDispatcherEvents();
}
@@ -514,7 +514,10 @@ public class TestApplication {
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, containerId);
Container c = mock(Container.class);
- when(c.getContainerID()).thenReturn(cId);
+ org.apache.hadoop.yarn.api.records.Container containerAPI =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(c.getContainer()).thenReturn(containerAPI);
+ when(c.getContainer().getId()).thenReturn(cId);
ContainerLaunchContext launchContext = mock(ContainerLaunchContext.class);
when(c.getLaunchContext()).thenReturn(launchContext);
when(launchContext.getApplicationACLs()).thenReturn(
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Wed Apr 24 02:21:58 2013
@@ -376,7 +376,7 @@ public class TestContainer {
public boolean matches(Object o) {
ContainersLauncherEvent evt = (ContainersLauncherEvent) o;
return evt.getType() == ContainersLauncherEventType.LAUNCH_CONTAINER
- && wcf.cId == evt.getContainer().getContainerID();
+ && wcf.cId == evt.getContainer().getContainer().getId();
}
};
verify(wc.launcherBus).handle(argThat(matchesLaunchReq));
@@ -639,7 +639,7 @@ public class TestContainer {
Path p = new Path(cache, rsrc.getKey());
localPaths.put(p, Arrays.asList(rsrc.getKey()));
// rsrc copied to p
- c.handle(new ContainerResourceLocalizedEvent(c.getContainerID(),
+ c.handle(new ContainerResourceLocalizedEvent(c.getContainer().getId(),
req, p));
}
drainDispatcherEvents();
@@ -662,7 +662,8 @@ public class TestContainer {
LocalResource rsrc = localResources.get(rsrcKey);
LocalResourceRequest req = new LocalResourceRequest(rsrc);
Exception e = new Exception("Fake localization error");
- c.handle(new ContainerResourceFailedEvent(c.getContainerID(), req, e));
+ c.handle(new ContainerResourceFailedEvent(c.getContainer()
+ .getId(), req, e));
drainDispatcherEvents();
}
@@ -677,7 +678,7 @@ public class TestContainer {
++counter;
LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
Exception e = new Exception("Fake localization error");
- c.handle(new ContainerResourceFailedEvent(c.getContainerID(),
+ c.handle(new ContainerResourceFailedEvent(c.getContainer().getId(),
req, e));
}
drainDispatcherEvents();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1471229&r1=1471228&r2=1471229&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Wed Apr 24 02:21:58 2013
@@ -37,6 +37,7 @@ import java.util.Map;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -51,14 +52,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
@@ -150,50 +150,17 @@ public class TestContainerLaunch extends
}
}
}
-
- // this is a dirty hack - but should be ok for a unittest.
- @SuppressWarnings({ "rawtypes", "unchecked" })
- public static void setNewEnvironmentHack(Map<String, String> newenv) throws Exception {
- try {
- Class<?> cl = Class.forName("java.lang.ProcessEnvironment");
- Field field = cl.getDeclaredField("theEnvironment");
- field.setAccessible(true);
- Map<String, String> env = (Map<String, String>)field.get(null);
- env.clear();
- env.putAll(newenv);
- Field ciField = cl.getDeclaredField("theCaseInsensitiveEnvironment");
- ciField.setAccessible(true);
- Map<String, String> cienv = (Map<String, String>)ciField.get(null);
- cienv.clear();
- cienv.putAll(newenv);
- } catch (NoSuchFieldException e) {
- Class[] classes = Collections.class.getDeclaredClasses();
- Map<String, String> env = System.getenv();
- for (Class cl : classes) {
- if ("java.util.Collections$UnmodifiableMap".equals(cl.getName())) {
- Field field = cl.getDeclaredField("m");
- field.setAccessible(true);
- Object obj = field.get(env);
- Map<String, String> map = (Map<String, String>) obj;
- map.clear();
- map.putAll(newenv);
- }
- }
- }
- }
/**
* See if environment variable is forwarded using sanitizeEnv.
* @throws Exception
*/
- @Test
+ @Test (timeout = 5000)
public void testContainerEnvVariables() throws Exception {
containerManager.start();
- Map<String, String> envWithDummy = new HashMap<String, String>();
- envWithDummy.putAll(System.getenv());
- envWithDummy.put(Environment.MALLOC_ARENA_MAX.name(), "99");
- setNewEnvironmentHack(envWithDummy);
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
@@ -207,34 +174,54 @@ public class TestContainerLaunch extends
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
cId.setApplicationAttemptId(appAttemptId);
- String malloc = System.getenv(Environment.MALLOC_ARENA_MAX.name());
+ when(mockContainer.getId()).thenReturn(cId);
+
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeHttpAddress()).thenReturn(
+ context.getNodeId().getHost() + ":12345");
+
+ Map<String, String> userSetEnv = new HashMap<String, String>();
+ userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
+ userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST");
+ userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
+ userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
+ userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
+ containerLaunchContext.setUser(user);
+ containerLaunchContext.setEnvironment(userSetEnv);
+
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile =
new File(tmpDir, "env_vars.txt").getAbsoluteFile();
if (Shell.WINDOWS) {
- fileWriter.println("@echo " + Environment.MALLOC_ARENA_MAX.$() + "> " +
- processStartFile);
- fileWriter.println("@echo " + cId + ">> " + processStartFile);
+ fileWriter.println("@echo " + Environment.CONTAINER_ID.$() + "> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.NM_HOST.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.NM_PORT.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.NM_HTTP_PORT.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.LOCAL_DIRS.$() + ">> "
+ + processStartFile);
fileWriter.println("@ping -n 100 127.0.0.1 >nul");
} else {
fileWriter.write("\numask 0"); // So that start file is readable by the test
- fileWriter.write("\necho " + Environment.MALLOC_ARENA_MAX.$() + " > " +
- processStartFile);
+ fileWriter.write("\necho $" + Environment.CONTAINER_ID.name() + " > "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.NM_HOST.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.NM_PORT.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.NM_HTTP_PORT.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.LOCAL_DIRS.name() + " >> "
+ + processStartFile);
fileWriter.write("\necho $$ >> " + processStartFile);
fileWriter.write("\nexec sleep 100");
}
fileWriter.close();
- assert(malloc != null && !"".equals(malloc));
-
- ContainerLaunchContext containerLaunchContext =
- recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
- when(mockContainer.getId()).thenReturn(cId);
-
- containerLaunchContext.setUser(user);
-
// upload the script file so that the container can run it
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -272,9 +259,40 @@ public class TestContainerLaunch extends
processStartFile.exists());
// Now verify the contents of the file
+ List<String> localDirs = dirsHandler.getLocalDirs();
+ List<Path> appDirs = new ArrayList<Path>(localDirs.size());
+ for (String localDir : localDirs) {
+ Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
+ Path userdir = new Path(usersdir, user);
+ Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
+ appDirs.add(new Path(appsdir, appId.toString()));
+ }
BufferedReader reader =
new BufferedReader(new FileReader(processStartFile));
- Assert.assertEquals(malloc, reader.readLine());
+ Assert.assertEquals(cId.toString(), reader.readLine());
+ Assert.assertEquals(mockContainer.getNodeId().getHost(),
+ reader.readLine());
+ Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()),
+ reader.readLine());
+ Assert.assertEquals(
+ String.valueOf(mockContainer.getNodeHttpAddress().split(":")[1]),
+ reader.readLine());
+ Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine());
+
+ Assert.assertEquals(cId.toString(), containerLaunchContext
+ .getEnvironment().get(Environment.CONTAINER_ID.name()));
+ Assert.assertEquals(mockContainer.getNodeId().getHost(),
+ containerLaunchContext.getEnvironment()
+ .get(Environment.NM_HOST.name()));
+ Assert.assertEquals(String.valueOf(mockContainer.getNodeId().getPort()),
+ containerLaunchContext.getEnvironment().get(
+ Environment.NM_PORT.name()));
+ Assert.assertEquals(
+ mockContainer.getNodeHttpAddress().split(":")[1],
+ containerLaunchContext.getEnvironment().get(
+ Environment.NM_HTTP_PORT.name()));
+ Assert.assertEquals(StringUtils.join(",", appDirs), containerLaunchContext
+ .getEnvironment().get(Environment.LOCAL_DIRS.name()));
// Get the pid of the process
String pid = reader.readLine().trim();
// No more lines
@@ -354,6 +372,9 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ when(mockContainer.getNodeHttpAddress()).thenReturn(
+ context.getNodeId().getHost() + ":12345");
containerLaunchContext.setUser(user);