You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2013/04/20 22:22:24 UTC
svn commit: r1470225 [1/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protoco...
Author: suresh
Date: Sat Apr 20 20:22:21 2013
New Revision: 1470225
URL: http://svn.apache.org/r1470225
Log:
Merge trunk to HDFS-2802 branch. This involves fixing many conflict with HDFS-4434.
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Sat Apr 20 20:22:21 2013
@@ -34,6 +34,13 @@ Trunk - Unreleased
YARN-487. Modify path manipulation in LocalDirsHandlerService to let
TestDiskFailures pass on Windows. (Chris Nauroth via vinodkv)
+ YARN-493. Fixed some shell related flaws in YARN on Windows. (Chris Nauroth
+ via vinodkv)
+
+ YARN-593. container launch on Windows does not correctly populate
+ classpath with new process's environment variables and localized resources
+ (Chris Nauroth via bikas)
+
BREAKDOWN OF HADOOP-8562 SUBTASKS
YARN-158. Yarn creating package-info.java must not depend on sh.
@@ -85,6 +92,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-444. Moved special container exit codes from YarnConfiguration to API
where they belong. (Sandy Ryza via vinodkv)
+ YARN-441. Removed unused utility methods for collections from two API
+ records. (Xuan Gong via vinodkv)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
@@ -151,6 +161,12 @@ Release 2.0.5-beta - UNRELEASED
YARN-514. Delayed store operations should not result in RM unavailability
for app submission (Zhijie Shen via bikas)
+ YARN-586. Fixed a typo in ApplicationSubmissionContext#setApplicationId.
+ (Zhijie Shen via vinodkv)
+
+ YARN-542. Changed the default global AM max-attempts value to be not one.
+ (Zhijie Shen via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
@@ -256,6 +272,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-585. Fix failure in TestFairScheduler#testNotAllowSubmitApplication
caused by YARN-514. (Zhijie Shen via vinodkv)
+ YARN-547. Fixed race conditions in public and private resource localization
+ which used to cause duplicate downloads. (Omkar Vinit Joshi via vinodkv)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java Sat Apr 20 20:22:21 2013
@@ -20,10 +20,8 @@ package org.apache.hadoop.yarn.api.proto
import java.util.List;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -120,36 +118,16 @@ public interface AllocateRequest {
@Stable
List<ResourceRequest> getAskList();
- @Private
- @Unstable
- ResourceRequest getAsk(int index);
-
- @Private
- @Unstable
- int getAskCount();
-
/**
- * Add list of <code>ResourceRequest</code> to update the
+ * Set list of <code>ResourceRequest</code> to update the
* <code>ResourceManager</code> about the application's resource requirements.
- * @param resourceRequest list of <code>ResourceRequest</code> to update the
+ * @param resourceRequests list of <code>ResourceRequest</code> to update the
* <code>ResourceManager</code> about the application's
* resource requirements
*/
@Public
@Stable
- void addAllAsks(List<ResourceRequest> resourceRequest);
-
- @Private
- @Unstable
- void addAsk(ResourceRequest request);
-
- @Private
- @Unstable
- void removeAsk(int index);
-
- @Private
- @Unstable
- void clearAsks();
+ void setAskList(List<ResourceRequest> resourceRequests);
/**
* Get the list of <code>ContainerId</code> of containers being
@@ -160,17 +138,9 @@ public interface AllocateRequest {
@Public
@Stable
List<ContainerId> getReleaseList();
-
- @Private
- @Unstable
- ContainerId getRelease(int index);
-
- @Private
- @Unstable
- int getReleaseCount();
/**
- * Add the list of <code>ContainerId</code> of containers being
+ * Set the list of <code>ContainerId</code> of containers being
* released by the <code>ApplicationMaster</code>
* @param releaseContainers list of <code>ContainerId</code> of
* containers being released by the <
@@ -178,17 +148,5 @@ public interface AllocateRequest {
*/
@Public
@Stable
- void addAllReleases(List<ContainerId> releaseContainers);
-
- @Private
- @Unstable
- void addRelease(ContainerId container);
-
- @Private
- @Unstable
- void removeRelease(int index);
-
- @Private
- @Unstable
- void clearReleases();
+ void setReleaseList(List<ContainerId> releaseContainers);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerResponse.java Sat Apr 20 20:22:21 2013
@@ -45,43 +45,11 @@ public interface StartContainerResponse
Map<String, ByteBuffer> getAllServiceResponse();
/**
- * Get the response from a single auxiliary service running on the
- * <code>NodeManager</code>
- *
- * @param key The auxiliary service name whose response is desired.
- * @return The opaque blob <code>ByteBuffer</code> returned by the auxiliary
- * service.
- */
- ByteBuffer getServiceResponse(String key);
-
- /**
- * Add to the list of auxiliary services which have been started on the
+ * Set to the list of auxiliary services which have been started on the
* <code>NodeManager</code>. This is done only once when the
* <code>NodeManager</code> starts up
- * @param serviceResponse A map from auxiliary service names to the opaque
+ * @param serviceResponses A map from auxiliary service names to the opaque
* blob <code>ByteBuffer</code>s for that auxiliary service
*/
- void addAllServiceResponse(Map<String, ByteBuffer> serviceResponse);
-
- /**
- * Add to the list of auxiliary services which have been started on the
- * <code>NodeManager</code>. This is done only once when the
- * <code>NodeManager</code> starts up
- *
- * @param key The auxiliary service name
- * @param value The opaque blob <code>ByteBuffer</code> managed by the
- * auxiliary service
- */
- void setServiceResponse(String key, ByteBuffer value);
-
- /**
- * Remove a single auxiliary service from the StartContainerResponse object
- * @param key The auxiliary service to remove
- */
- void removeServiceResponse(String key);
-
- /**
- * Remove all the auxiliary services from the StartContainerResponse object
- */
- void clearServiceResponse();
+ void setAllServiceResponse(Map<String, ByteBuffer> serviceResponses);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java Sat Apr 20 20:22:21 2013
@@ -25,7 +25,6 @@ import java.util.List;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -144,14 +143,13 @@ public class AllocateRequestPBImpl exten
return this.ask;
}
@Override
- public ResourceRequest getAsk(int index) {
- initAsks();
- return this.ask.get(index);
- }
- @Override
- public int getAskCount() {
+ public void setAskList(final List<ResourceRequest> resourceRequests) {
+ if(resourceRequests == null) {
+ return;
+ }
initAsks();
- return this.ask.size();
+ this.ask.clear();
+ this.ask.addAll(resourceRequests);
}
private void initAsks() {
@@ -167,14 +165,6 @@ public class AllocateRequestPBImpl exten
}
}
- @Override
- public void addAllAsks(final List<ResourceRequest> ask) {
- if (ask == null)
- return;
- initAsks();
- this.ask.addAll(ask);
- }
-
private void addAsksToProto() {
maybeInitBuilder();
builder.clearAsk();
@@ -209,34 +199,18 @@ public class AllocateRequestPBImpl exten
builder.addAllAsk(iterable);
}
@Override
- public void addAsk(ResourceRequest ask) {
- initAsks();
- this.ask.add(ask);
- }
- @Override
- public void removeAsk(int index) {
- initAsks();
- this.ask.remove(index);
- }
- @Override
- public void clearAsks() {
- initAsks();
- this.ask.clear();
- }
- @Override
public List<ContainerId> getReleaseList() {
initReleases();
return this.release;
}
@Override
- public ContainerId getRelease(int index) {
- initReleases();
- return this.release.get(index);
- }
- @Override
- public int getReleaseCount() {
+ public void setReleaseList(List<ContainerId> releaseContainers) {
+ if(releaseContainers == null) {
+ return;
+ }
initReleases();
- return this.release.size();
+ this.release.clear();
+ this.release.addAll(releaseContainers);
}
private void initReleases() {
@@ -252,14 +226,6 @@ public class AllocateRequestPBImpl exten
}
}
- @Override
- public void addAllReleases(final List<ContainerId> release) {
- if (release == null)
- return;
- initReleases();
- this.release.addAll(release);
- }
-
private void addReleasesToProto() {
maybeInitBuilder();
builder.clearRelease();
@@ -293,21 +259,6 @@ public class AllocateRequestPBImpl exten
};
builder.addAllRelease(iterable);
}
- @Override
- public void addRelease(ContainerId release) {
- initReleases();
- this.release.add(release);
- }
- @Override
- public void removeRelease(int index) {
- initReleases();
- this.release.remove(index);
- }
- @Override
- public void clearReleases() {
- initReleases();
- this.release.clear();
- }
private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
return new ApplicationAttemptIdPBImpl(p);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerResponsePBImpl.java Sat Apr 20 20:22:21 2013
@@ -84,9 +84,14 @@ public class StartContainerResponsePBImp
return this.serviceResponse;
}
@Override
- public synchronized ByteBuffer getServiceResponse(String key) {
+ public synchronized void setAllServiceResponse(
+ Map<String, ByteBuffer> serviceResponses) {
+ if(serviceResponses == null) {
+ return;
+ }
initServiceResponse();
- return this.serviceResponse.get(key);
+ this.serviceResponse.clear();
+ this.serviceResponse.putAll(serviceResponses);
}
private synchronized void initServiceResponse() {
@@ -102,14 +107,6 @@ public class StartContainerResponsePBImp
}
}
- @Override
- public synchronized void addAllServiceResponse(final Map<String, ByteBuffer> serviceResponse) {
- if (serviceResponse == null)
- return;
- initServiceResponse();
- this.serviceResponse.putAll(serviceResponse);
- }
-
private synchronized void addServiceResponseToProto() {
maybeInitBuilder();
builder.clearServiceResponse();
@@ -143,19 +140,4 @@ public class StartContainerResponsePBImp
};
builder.addAllServiceResponse(iterable);
}
- @Override
- public synchronized void setServiceResponse(String key, ByteBuffer val) {
- initServiceResponse();
- this.serviceResponse.put(key, val);
- }
- @Override
- public synchronized void removeServiceResponse(String key) {
- initServiceResponse();
- this.serviceResponse.remove(key);
- }
- @Override
- public synchronized void clearServiceResponse() {
- initServiceResponse();
- this.serviceResponse.clear();
- }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java Sat Apr 20 20:22:21 2013
@@ -58,12 +58,12 @@ public interface ApplicationSubmissionCo
/**
* Set the <code>ApplicationId</code> of the submitted application.
- * @param appplicationId <code>ApplicationId</code> of the submitted
- * application
+ * @param applicationId <code>ApplicationId</code> of the submitted
+ * application
*/
@Public
@Stable
- public void setApplicationId(ApplicationId appplicationId);
+ public void setApplicationId(ApplicationId applicationId);
/**
* Get the application <em>name</em>.
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sat Apr 20 20:22:21 2013
@@ -186,7 +186,7 @@ public class YarnConfiguration extends C
*/
public static final String RM_AM_MAX_ATTEMPTS =
RM_PREFIX + "am.max-attempts";
- public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 1;
+ public static final int DEFAULT_RM_AM_MAX_ATTEMPTS = 2;
/** The keytab for the resource manager.*/
public static final String RM_KEYTAB =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Sat Apr 20 20:22:21 2013
@@ -393,8 +393,8 @@ public class BuilderUtils {
allocateRequest.setApplicationAttemptId(applicationAttemptId);
allocateRequest.setResponseId(responseID);
allocateRequest.setProgress(appProgress);
- allocateRequest.addAllAsks(resourceAsk);
- allocateRequest.addAllReleases(containersToBeReleased);
+ allocateRequest.setAskList(resourceAsk);
+ allocateRequest.setReleaseList(containersToBeReleased);
return allocateRequest;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sat Apr 20 20:22:21 2013
@@ -145,9 +145,10 @@
setting for all application masters. Each application master can specify
its individual maximum number of application attempts via the API, but the
individual number cannot be more than the global upper bound. If it is,
- the resourcemanager will override it.</description>
+ the resourcemanager will override it. The default number is set to 2, to
+ allow at least one retry for AM.</description>
<name>yarn.resourcemanager.am.max-attempts</name>
- <value>1</value>
+ <value>2</value>
</property>
<property>
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Sat Apr 20 20:22:21 2013
@@ -222,19 +222,6 @@ public abstract class ContainerExecutor
}
- /** Return a command for determining if process with specified pid is alive. */
- protected static String[] getCheckProcessIsAliveCommand(String pid) {
- return Shell.WINDOWS ?
- new String[] { Shell.WINUTILS, "task", "isAlive", pid } :
- new String[] { "kill", "-0", pid };
- }
-
- /** Return a command to send a signal to a given pid */
- protected static String[] getSignalKillCommand(int code, String pid) {
- return Shell.WINDOWS ? new String[] { Shell.WINUTILS, "task", "kill", pid } :
- new String[] { "kill", "-" + code, pid };
- }
-
/**
* Is the container still active?
* @param containerId
@@ -303,26 +290,6 @@ public abstract class ContainerExecutor
return pid;
}
- public static final boolean isSetsidAvailable = isSetsidSupported();
- private static boolean isSetsidSupported() {
- if (Shell.WINDOWS) {
- return true;
- }
- ShellCommandExecutor shexec = null;
- boolean setsidSupported = true;
- try {
- String[] args = {"setsid", "bash", "-c", "echo $$"};
- shexec = new ShellCommandExecutor(args);
- shexec.execute();
- } catch (IOException ioe) {
- LOG.warn("setsid is not available on this machine. So not using it.");
- setsidSupported = false;
- } finally { // handle the exit code
- LOG.info("setsid exited with exit code " + shexec.getExitCode());
- }
- return setsidSupported;
- }
-
public static class DelayedProcessKiller extends Thread {
private final String user;
private final String pid;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Sat Apr 20 20:22:21 2013
@@ -50,6 +50,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.annotations.VisibleForTesting;
+
public class DefaultContainerExecutor extends ContainerExecutor {
private static final Log LOG = LogFactory
@@ -237,8 +239,9 @@ public class DefaultContainerExecutor ex
protected abstract void writeLocalWrapperScript(Path launchDst, Path pidFile,
PrintStream pout);
- protected LocalWrapperScriptBuilder(Path wrapperScriptPath) {
- this.wrapperScriptPath = wrapperScriptPath;
+ protected LocalWrapperScriptBuilder(Path containerWorkDir) {
+ this.wrapperScriptPath = new Path(containerWorkDir,
+ Shell.appendScriptExtension("default_container_executor"));
}
}
@@ -246,7 +249,7 @@ public class DefaultContainerExecutor ex
extends LocalWrapperScriptBuilder {
public UnixLocalWrapperScriptBuilder(Path containerWorkDir) {
- super(new Path(containerWorkDir, "default_container_executor.sh"));
+ super(containerWorkDir);
}
@Override
@@ -260,7 +263,7 @@ public class DefaultContainerExecutor ex
pout.println();
pout.println("echo $$ > " + pidFile.toString() + ".tmp");
pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile);
- String exec = ContainerExecutor.isSetsidAvailable? "exec setsid" : "exec";
+ String exec = Shell.isSetsidAvailable? "exec setsid" : "exec";
pout.println(exec + " /bin/bash -c \"" +
launchDst.toUri().getPath().toString() + "\"");
}
@@ -274,7 +277,7 @@ public class DefaultContainerExecutor ex
public WindowsLocalWrapperScriptBuilder(String containerIdStr,
Path containerWorkDir) {
- super(new Path(containerWorkDir, "default_container_executor.cmd"));
+ super(containerWorkDir);
this.containerIdStr = containerIdStr;
}
@@ -297,18 +300,15 @@ public class DefaultContainerExecutor ex
@Override
public boolean signalContainer(String user, String pid, Signal signal)
throws IOException {
- final String sigpid = ContainerExecutor.isSetsidAvailable
- ? "-" + pid
- : pid;
- LOG.debug("Sending signal " + signal.getValue() + " to pid " + sigpid
+ LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid
+ " as user " + user);
- if (!containerIsAlive(sigpid)) {
+ if (!containerIsAlive(pid)) {
return false;
}
try {
- killContainer(sigpid, signal);
+ killContainer(pid, signal);
} catch (IOException e) {
- if (!containerIsAlive(sigpid)) {
+ if (!containerIsAlive(pid)) {
return false;
}
throw e;
@@ -322,9 +322,11 @@ public class DefaultContainerExecutor ex
* @param pid String pid
* @return boolean true if the process is alive
*/
- private boolean containerIsAlive(String pid) throws IOException {
+ @VisibleForTesting
+ public static boolean containerIsAlive(String pid) throws IOException {
try {
- new ShellCommandExecutor(getCheckProcessIsAliveCommand(pid)).execute();
+ new ShellCommandExecutor(Shell.getCheckProcessIsAliveCommand(pid))
+ .execute();
// successful execution means process is alive
return true;
}
@@ -342,7 +344,7 @@ public class DefaultContainerExecutor ex
* (for logging).
*/
private void killContainer(String pid, Signal signal) throws IOException {
- new ShellCommandExecutor(getSignalKillCommand(signal.getValue(), pid))
+ new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid))
.execute();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Sat Apr 20 20:22:21 2013
@@ -468,7 +468,7 @@ public class ContainerManagerImpl extend
StartContainerResponse response =
recordFactory.newRecordInstance(StartContainerResponse.class);
- response.addAllServiceResponse(auxiliaryServices.getMeta());
+ response.setAllServiceResponse(auxiliaryServices.getMeta());
// TODO launchedContainer misplaced -> doesn't necessarily mean a container
// launch. A finished Application will not launch containers.
metrics.launchedContainer();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Sat Apr 20 20:22:21 2013
@@ -28,6 +28,7 @@ import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -72,8 +73,8 @@ public class ContainerLaunch implements
private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
- public static final String CONTAINER_SCRIPT = Shell.WINDOWS ?
- "launch_container.cmd" : "launch_container.sh";
+ public static final String CONTAINER_SCRIPT =
+ Shell.appendScriptExtension("launch_container");
public static final String FINAL_CONTAINER_TOKENS_FILE = "container_tokens";
private static final String PID_FILE_NAME_FMT = "%s.pid";
@@ -211,7 +212,7 @@ public class ContainerLaunch implements
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
// Sanitize the container's environment
- sanitizeEnv(environment, containerWorkDir, appDirs);
+ sanitizeEnv(environment, containerWorkDir, appDirs, localResources);
// Write out the environment
writeLaunchEnv(containerScriptOutStream, environment, localResources,
@@ -506,9 +507,17 @@ public class ContainerLaunch implements
@Override
protected void link(Path src, Path dst) throws IOException {
- line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS,
- new File(dst.toString()).getPath(),
- new File(src.toUri().getPath()).getPath()));
+ File srcFile = new File(src.toUri().getPath());
+ String srcFileStr = srcFile.getPath();
+ String dstFileStr = new File(dst.toString()).getPath();
+ // If not on Java7+ on Windows, then copy file instead of symlinking.
+ // See also FileUtil#symLink for full explanation.
+ if (!Shell.isJava7OrAbove() && srcFile.isFile()) {
+ line(String.format("@copy \"%s\" \"%s\"", srcFileStr, dstFileStr));
+ } else {
+ line(String.format("@%s symlink \"%s\" \"%s\"", Shell.WINUTILS,
+ dstFileStr, srcFileStr));
+ }
}
@Override
@@ -532,7 +541,8 @@ public class ContainerLaunch implements
}
public void sanitizeEnv(Map<String, String> environment,
- Path pwd, List<Path> appDirs) throws IOException {
+ Path pwd, List<Path> appDirs, Map<Path, List<String>> resources)
+ throws IOException {
/**
* Non-modifiable environment variables
*/
@@ -566,16 +576,6 @@ public class ContainerLaunch implements
environment.put("JVM_PID", "$$");
}
- // TODO: Remove Windows check and use this approach on all platforms after
- // additional testing. See YARN-358.
- if (Shell.WINDOWS) {
- String inputClassPath = environment.get(Environment.CLASSPATH.name());
- if (inputClassPath != null && !inputClassPath.isEmpty()) {
- environment.put(Environment.CLASSPATH.name(),
- FileUtil.createJarWithClassPath(inputClassPath, pwd));
- }
- }
-
/**
* Modifiable environment variables
*/
@@ -594,6 +594,57 @@ public class ContainerLaunch implements
YarnConfiguration.NM_ADMIN_USER_ENV,
YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV)
);
+
+ // TODO: Remove Windows check and use this approach on all platforms after
+ // additional testing. See YARN-358.
+ if (Shell.WINDOWS) {
+ String inputClassPath = environment.get(Environment.CLASSPATH.name());
+ if (inputClassPath != null && !inputClassPath.isEmpty()) {
+ StringBuilder newClassPath = new StringBuilder(inputClassPath);
+
+ // Localized resources do not exist at the desired paths yet, because the
+ // container launch script has not run to create symlinks yet. This
+ // means that FileUtil.createJarWithClassPath can't automatically expand
+ // wildcards to separate classpath entries for each file in the manifest.
+ // To resolve this, append classpath entries explicitly for each
+ // resource.
+ for (Map.Entry<Path,List<String>> entry : resources.entrySet()) {
+ boolean targetIsDirectory = new File(entry.getKey().toUri().getPath())
+ .isDirectory();
+
+ for (String linkName : entry.getValue()) {
+ // Append resource.
+ newClassPath.append(File.pathSeparator).append(pwd.toString())
+ .append(Path.SEPARATOR).append(linkName);
+
+ // FileUtil.createJarWithClassPath must use File.toURI to convert
+ // each file to a URI to write into the manifest's classpath. For
+ // directories, the classpath must have a trailing '/', but
+ // File.toURI only appends the trailing '/' if it is a directory that
+ // already exists. To resolve this, add the classpath entries with
+ // explicit trailing '/' here for any localized resource that targets
+ // a directory. Then, FileUtil.createJarWithClassPath will guarantee
+ // that the resulting entry in the manifest's classpath will have a
+ // trailing '/', and thus refer to a directory instead of a file.
+ if (targetIsDirectory) {
+ newClassPath.append(Path.SEPARATOR);
+ }
+ }
+ }
+
+ // When the container launches, it takes the parent process's environment
+ // and then adds/overwrites with the entries from the container launch
+ // context. Do the same thing here for correct substitution of
+ // environment variables in the classpath jar manifest.
+ Map<String, String> mergedEnv = new HashMap<String, String>(
+ System.getenv());
+ mergedEnv.putAll(environment);
+
+ String classPathJar = FileUtil.createJarWithClassPath(
+ newClassPath.toString(), pwd, mergedEnv);
+ environment.put(Environment.CLASSPATH.name(), classPathJar);
+ }
+ }
}
static void writeLaunchEnv(OutputStream out,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java Sat Apr 20 20:22:21 2013
@@ -18,12 +18,15 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Component tracking resources all of the same {@link LocalResourceVisibility}
*
@@ -41,4 +44,8 @@ interface LocalResourcesTracker
String getUser();
long nextUniqueNumber();
+
+ @VisibleForTesting
+ @Private
+ LocalizedResource getLocalizedResource(LocalResourceRequest request);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java Sat Apr 20 20:22:21 2013
@@ -27,6 +27,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -35,6 +36,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* A collection of {@link LocalizedResource}s all of same
@@ -307,4 +310,11 @@ class LocalResourcesTrackerImpl implemen
public long nextUniqueNumber() {
return uniqueNumberGenerator.incrementAndGet();
}
+
+ @VisibleForTesting
+ @Private
+ @Override
+ public LocalizedResource getLocalizedResource(LocalResourceRequest request) {
+ return localrsrc.get(request);
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Sat Apr 20 20:22:21 2013
@@ -78,19 +78,14 @@ public class LocalizedResource implement
// From INIT (ref == 0, awaiting req)
.addTransition(ResourceState.INIT, ResourceState.DOWNLOADING,
ResourceEventType.REQUEST, new FetchResourceTransition())
- .addTransition(ResourceState.INIT, ResourceState.LOCALIZED,
- ResourceEventType.LOCALIZED, new FetchDirectTransition())
- .addTransition(ResourceState.INIT, ResourceState.INIT,
- ResourceEventType.RELEASE, new ReleaseTransition())
// From DOWNLOADING (ref > 0, may be localizing)
.addTransition(ResourceState.DOWNLOADING, ResourceState.DOWNLOADING,
ResourceEventType.REQUEST, new FetchResourceTransition()) // TODO: Duplicate addition!!
.addTransition(ResourceState.DOWNLOADING, ResourceState.LOCALIZED,
ResourceEventType.LOCALIZED, new FetchSuccessTransition())
- .addTransition(ResourceState.DOWNLOADING,
- EnumSet.of(ResourceState.DOWNLOADING, ResourceState.INIT),
- ResourceEventType.RELEASE, new ReleasePendingTransition())
+ .addTransition(ResourceState.DOWNLOADING,ResourceState.DOWNLOADING,
+ ResourceEventType.RELEASE, new ReleaseTransition())
.addTransition(ResourceState.DOWNLOADING, ResourceState.FAILED,
ResourceEventType.LOCALIZATION_FAILED, new FetchFailedTransition())
@@ -98,8 +93,6 @@ public class LocalizedResource implement
.addTransition(ResourceState.LOCALIZED, ResourceState.LOCALIZED,
ResourceEventType.REQUEST, new LocalizedResourceTransition())
.addTransition(ResourceState.LOCALIZED, ResourceState.LOCALIZED,
- ResourceEventType.LOCALIZED)
- .addTransition(ResourceState.LOCALIZED, ResourceState.LOCALIZED,
ResourceEventType.RELEASE, new ReleaseTransition())
.installTopology();
@@ -230,14 +223,6 @@ public class LocalizedResource implement
}
}
- private static class FetchDirectTransition extends FetchSuccessTransition {
- @Override
- public void transition(LocalizedResource rsrc, ResourceEvent event) {
- LOG.warn("Resource " + rsrc + " localized without listening container");
- super.transition(rsrc, event);
- }
- }
-
/**
* Resource localized, notify waiting containers.
*/
@@ -304,17 +289,4 @@ public class LocalizedResource implement
rsrc.release(relEvent.getContainer());
}
}
-
- private static class ReleasePendingTransition implements
- MultipleArcTransition<LocalizedResource,ResourceEvent,ResourceState> {
- @Override
- public ResourceState transition(LocalizedResource rsrc,
- ResourceEvent event) {
- ResourceReleaseEvent relEvent = (ResourceReleaseEvent) event;
- rsrc.release(relEvent.getContainer());
- return rsrc.ref.isEmpty()
- ? ResourceState.INIT
- : ResourceState.DOWNLOADING;
- }
- }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Sat Apr 20 20:22:21 2013
@@ -31,7 +31,6 @@ import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CancellationException;
@@ -47,9 +46,11 @@ import java.util.concurrent.ScheduledExe
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
@@ -112,6 +113,7 @@ import org.apache.hadoop.yarn.service.Co
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ResourceLocalizationService extends CompositeService
@@ -492,7 +494,25 @@ public class ResourceLocalizationService
+ Path.SEPARATOR + appId;
return path;
}
+
+ @VisibleForTesting
+ @Private
+ public PublicLocalizer getPublicLocalizer() {
+ return localizerTracker.publicLocalizer;
+ }
+ @VisibleForTesting
+ @Private
+ public LocalizerRunner getLocalizerRunner(String locId) {
+ return localizerTracker.privLocalizers.get(locId);
+ }
+
+ @VisibleForTesting
+ @Private
+ public Map<String, LocalizerRunner> getPrivateLocalizers() {
+ return localizerTracker.privLocalizers;
+ }
+
/**
* Sub-component handling the spawning of {@link ContainerLocalizer}s
*/
@@ -606,41 +626,20 @@ public class ResourceLocalizationService
final ExecutorService threadPool;
final CompletionService<Path> queue;
final Map<Future<Path>,LocalizerResourceRequestEvent> pending;
- // TODO hack to work around broken signaling
- final Map<LocalResourceRequest,List<LocalizerResourceRequestEvent>> attempts;
PublicLocalizer(Configuration conf) {
this(conf, getLocalFileContext(conf),
createLocalizerExecutor(conf),
- new HashMap<Future<Path>,LocalizerResourceRequestEvent>(),
- new HashMap<LocalResourceRequest,List<LocalizerResourceRequestEvent>>());
+ new HashMap<Future<Path>,LocalizerResourceRequestEvent>());
}
PublicLocalizer(Configuration conf, FileContext lfs,
ExecutorService threadPool,
- Map<Future<Path>,LocalizerResourceRequestEvent> pending,
- Map<LocalResourceRequest,List<LocalizerResourceRequestEvent>> attempts) {
+ Map<Future<Path>,LocalizerResourceRequestEvent> pending) {
super("Public Localizer");
this.lfs = lfs;
this.conf = conf;
this.pending = pending;
- this.attempts = attempts;
-// List<String> localDirs = dirsHandler.getLocalDirs();
-// String[] publicFilecache = new String[localDirs.size()];
-// for (int i = 0, n = localDirs.size(); i < n; ++i) {
-// publicFilecache[i] =
-// new Path(localDirs.get(i), ContainerLocalizer.FILECACHE).toString();
-// }
-// conf.setStrings(PUBCACHE_CTXT, publicFilecache);
-
-// this.publicDirDestPath = new LocalDirAllocator(PUBCACHE_CTXT).getLocalPathForWrite(pathStr, conf);
-// List<String> localDirs = dirsHandler.getLocalDirs();
-// String[] publicFilecache = new String[localDirs.size()];
-// int i = 0;
-// for (String localDir : localDirs) {
-// publicFilecache[i++] =
-// new Path(localDir, ContainerLocalizer.FILECACHE).toString();
-// }
this.threadPool = threadPool;
this.queue = new ExecutorCompletionService<Path>(threadPool);
@@ -648,36 +647,45 @@ public class ResourceLocalizationService
public void addResource(LocalizerResourceRequestEvent request) {
// TODO handle failures, cancellation, requests by other containers
- LocalResourceRequest key = request.getResource().getRequest();
+ LocalizedResource rsrc = request.getResource();
+ LocalResourceRequest key = rsrc.getRequest();
LOG.info("Downloading public rsrc:" + key);
- synchronized (attempts) {
- List<LocalizerResourceRequestEvent> sigh = attempts.get(key);
- if (null == sigh) {
+ /*
+ * Here multiple containers may request the same resource. So we need
+ * to start downloading only when
+ * 1) ResourceState == DOWNLOADING
+ * 2) We are able to acquire non blocking semaphore lock.
+ * If not we will skip this resource as either it is getting downloaded
+ * or it FAILED / LOCALIZED.
+ */
+
+ if (rsrc.tryAcquire()) {
+ if (rsrc.getState().equals(ResourceState.DOWNLOADING)) {
LocalResource resource = request.getResource().getRequest();
try {
- Path publicDirDestPath = dirsHandler.getLocalPathForWrite(
- "." + Path.SEPARATOR + ContainerLocalizer.FILECACHE,
- ContainerLocalizer.getEstimatedSize(resource), true);
+ Path publicDirDestPath =
+ dirsHandler.getLocalPathForWrite("." + Path.SEPARATOR
+ + ContainerLocalizer.FILECACHE,
+ ContainerLocalizer.getEstimatedSize(resource), true);
Path hierarchicalPath =
- publicRsrc.getPathForLocalization(key, publicDirDestPath);
+ publicRsrc.getPathForLocalization(key, publicDirDestPath);
if (!hierarchicalPath.equals(publicDirDestPath)) {
publicDirDestPath = hierarchicalPath;
- DiskChecker.checkDir(
- new File(publicDirDestPath.toUri().getPath()));
+ DiskChecker.checkDir(new File(publicDirDestPath.toUri().getPath()));
}
publicDirDestPath =
new Path(publicDirDestPath, Long.toString(publicRsrc
.nextUniqueNumber()));
- pending.put(queue.submit(new FSDownload(
- lfs, null, conf, publicDirDestPath, resource)),
- request);
- attempts.put(key, new LinkedList<LocalizerResourceRequestEvent>());
+ pending.put(queue.submit(new FSDownload(lfs, null, conf,
+ publicDirDestPath, resource)), request);
} catch (IOException e) {
+ rsrc.unlock();
+ // TODO Need to Fix IO Exceptions - Notifying resource
LOG.error("Local path for public localization is not found. "
+ " May be disks failed.", e);
}
} else {
- sigh.add(request);
+ rsrc.unlock();
}
}
}
@@ -700,24 +708,14 @@ public class ResourceLocalizationService
LocalResourceRequest key = assoc.getResource().getRequest();
publicRsrc.handle(new ResourceLocalizedEvent(key, local, FileUtil
.getDU(new File(local.toUri()))));
- synchronized (attempts) {
- attempts.remove(key);
- }
+ assoc.getResource().unlock();
} catch (ExecutionException e) {
LOG.info("Failed to download rsrc " + assoc.getResource(),
e.getCause());
LocalResourceRequest req = assoc.getResource().getRequest();
publicRsrc.handle(new ResourceFailedLocalizationEvent(req, e
.getCause()));
- synchronized (attempts) {
- List<LocalizerResourceRequestEvent> reqs;
- reqs = attempts.get(req);
- if (null == reqs) {
- LOG.error("Missing pending list for " + req);
- return;
- }
- attempts.remove(req);
- }
+ assoc.getResource().unlock();
} catch (CancellationException e) {
// ignore; shutting down
}
@@ -776,22 +774,35 @@ public class ResourceLocalizationService
i.hasNext();) {
LocalizerResourceRequestEvent evt = i.next();
LocalizedResource nRsrc = evt.getResource();
- if (ResourceState.LOCALIZED.equals(nRsrc.getState())) {
+ // Resource download should take place ONLY if resource is in
+ // Downloading state
+ if (!ResourceState.DOWNLOADING.equals(nRsrc.getState())) {
i.remove();
continue;
}
+ /*
+ * Multiple containers will try to download the same resource. So the
+ * resource download should start only if
+ * 1) We can acquire a non blocking semaphore lock on resource
+ * 2) Resource is still in DOWNLOADING state
+ */
if (nRsrc.tryAcquire()) {
- LocalResourceRequest nextRsrc = nRsrc.getRequest();
- LocalResource next =
- recordFactory.newRecordInstance(LocalResource.class);
- next.setResource(
- ConverterUtils.getYarnUrlFromPath(nextRsrc.getPath()));
- next.setTimestamp(nextRsrc.getTimestamp());
- next.setType(nextRsrc.getType());
- next.setVisibility(evt.getVisibility());
- next.setPattern(evt.getPattern());
- scheduled.put(nextRsrc, evt);
- return next;
+ if (nRsrc.getState().equals(ResourceState.DOWNLOADING)) {
+ LocalResourceRequest nextRsrc = nRsrc.getRequest();
+ LocalResource next =
+ recordFactory.newRecordInstance(LocalResource.class);
+ next.setResource(ConverterUtils.getYarnUrlFromPath(nextRsrc
+ .getPath()));
+ next.setTimestamp(nextRsrc.getTimestamp());
+ next.setType(nextRsrc.getType());
+ next.setVisibility(evt.getVisibility());
+ next.setPattern(evt.getPattern());
+ scheduled.put(nextRsrc, evt);
+ return next;
+ } else {
+ // Need to release acquired lock
+ nRsrc.unlock();
+ }
}
}
return null;
@@ -863,6 +874,12 @@ public class ResourceLocalizationService
new ResourceLocalizedEvent(req, ConverterUtils
.getPathFromYarnURL(stat.getLocalPath()), stat.getLocalSize()));
} catch (URISyntaxException e) { }
+
+ // unlocking the resource and removing it from scheduled resource
+ // list
+ assoc.getResource().unlock();
+ scheduled.remove(req);
+
if (pending.isEmpty()) {
// TODO: Synchronization
response.setLocalizerAction(LocalizerAction.DIE);
@@ -889,11 +906,16 @@ public class ResourceLocalizationService
break;
case FETCH_FAILURE:
LOG.info("DEBUG: FAILED " + req, stat.getException());
- assoc.getResource().unlock();
response.setLocalizerAction(LocalizerAction.DIE);
getLocalResourcesTracker(req.getVisibility(), user, applicationId)
.handle(
new ResourceFailedLocalizationEvent(req, stat.getException()));
+
+ // unlocking the resource and removing it from scheduled resource
+ // list
+ assoc.getResource().unlock();
+ scheduled.remove(req);
+
break;
default:
LOG.info("Unknown status: " + stat.getStatus());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Sat Apr 20 20:22:21 2013
@@ -22,12 +22,13 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -40,6 +41,7 @@ import junit.framework.Assert;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -59,6 +61,7 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -81,6 +84,7 @@ public class TestNodeManagerShutdown {
.getRecordFactory(null);
static final String user = "nobody";
private FileContext localFS;
+ private ContainerId cId;
private CyclicBarrier syncBarrier = new CyclicBarrier(2);
@Before
@@ -90,6 +94,9 @@ public class TestNodeManagerShutdown {
logsDir.mkdirs();
remoteLogsDir.mkdirs();
nmLocalDir.mkdirs();
+
+ // Construct the Container-id
+ cId = createContainerId();
}
@After
@@ -115,25 +122,32 @@ public class TestNodeManagerShutdown {
nm.stop();
- // Now verify the contents of the file
- // Script generates a message when it receives a sigterm
- // so we look for that
- BufferedReader reader =
- new BufferedReader(new FileReader(processStartFile));
-
- boolean foundSigTermMessage = false;
- while (true) {
- String line = reader.readLine();
- if (line == null) {
- break;
- }
- if (line.contains("SIGTERM")) {
- foundSigTermMessage = true;
- break;
+ // Now verify the contents of the file. Script generates a message when it
+ // receives a sigterm so we look for that. We cannot perform this check on
+ // Windows, because the process is not notified when killed by winutils.
+ // There is no way for the process to trap and respond. Instead, we can
+ // verify that the job object with ID matching container ID no longer exists.
+ if (Shell.WINDOWS) {
+ Assert.assertFalse("Process is still alive!",
+ DefaultContainerExecutor.containerIsAlive(cId.toString()));
+ } else {
+ BufferedReader reader =
+ new BufferedReader(new FileReader(processStartFile));
+
+ boolean foundSigTermMessage = false;
+ while (true) {
+ String line = reader.readLine();
+ if (line == null) {
+ break;
+ }
+ if (line.contains("SIGTERM")) {
+ foundSigTermMessage = true;
+ break;
+ }
}
+ Assert.assertTrue("Did not find sigterm message", foundSigTermMessage);
+ reader.close();
}
- Assert.assertTrue("Did not find sigterm message", foundSigTermMessage);
- reader.close();
}
@SuppressWarnings("unchecked")
@@ -162,8 +176,6 @@ public class TestNodeManagerShutdown {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
Container mockContainer = mock(Container.class);
- // Construct the Container-id
- ContainerId cId = createContainerId();
when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -184,9 +196,7 @@ public class TestNodeManagerShutdown {
localResources.put(destinationFile, localResource);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
- List<String> commands = new ArrayList<String>();
- commands.add("/bin/bash");
- commands.add(scriptFile.getAbsolutePath());
+ List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource resource = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(resource);
@@ -234,16 +244,24 @@ public class TestNodeManagerShutdown {
* stopped by external means.
*/
private File createUnhaltingScriptFile() throws IOException {
- File scriptFile = new File(tmpDir, "scriptFile.sh");
- BufferedWriter fileWriter = new BufferedWriter(new FileWriter(scriptFile));
- fileWriter.write("#!/bin/bash\n\n");
- fileWriter.write("echo \"Running testscript for delayed kill\"\n");
- fileWriter.write("hello=\"Got SIGTERM\"\n");
- fileWriter.write("umask 0\n");
- fileWriter.write("trap \"echo $hello >> " + processStartFile + "\" SIGTERM\n");
- fileWriter.write("echo \"Writing pid to start file\"\n");
- fileWriter.write("echo $$ >> " + processStartFile + "\n");
- fileWriter.write("while true; do\ndate >> /dev/null;\n done\n");
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
+ PrintWriter fileWriter = new PrintWriter(scriptFile);
+ if (Shell.WINDOWS) {
+ fileWriter.println("@echo \"Running testscript for delayed kill\"");
+ fileWriter.println("@echo \"Writing pid to start file\"");
+ fileWriter.println("@echo " + cId + ">> " + processStartFile);
+ fileWriter.println("@pause");
+ } else {
+ fileWriter.write("#!/bin/bash\n\n");
+ fileWriter.write("echo \"Running testscript for delayed kill\"\n");
+ fileWriter.write("hello=\"Got SIGTERM\"\n");
+ fileWriter.write("umask 0\n");
+ fileWriter.write("trap \"echo $hello >> " + processStartFile +
+ "\" SIGTERM\n");
+ fileWriter.write("echo \"Writing pid to start file\"\n");
+ fileWriter.write("echo $$ >> " + processStartFile + "\n");
+ fileWriter.write("while true; do\ndate >> /dev/null;\n done\n");
+ }
fileWriter.close();
return scriptFile;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Sat Apr 20 20:22:21 2013
@@ -76,15 +76,15 @@ public abstract class BaseContainerManag
public BaseContainerManagerTest() throws UnsupportedFileSystemException {
localFS = FileContext.getLocalFSFileContext();
localDir =
- new File("target", this.getClass().getName() + "-localDir")
+ new File("target", this.getClass().getSimpleName() + "-localDir")
.getAbsoluteFile();
localLogDir =
- new File("target", this.getClass().getName() + "-localLogDir")
+ new File("target", this.getClass().getSimpleName() + "-localLogDir")
.getAbsoluteFile();
remoteLogDir =
- new File("target", this.getClass().getName() + "-remoteLogDir")
+ new File("target", this.getClass().getSimpleName() + "-remoteLogDir")
.getAbsoluteFile();
- tmpDir = new File("target", this.getClass().getName() + "-tmpDir");
+ tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir");
}
protected static Log LOG = LogFactory
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1470225&r1=1470224&r2=1470225&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Sat Apr 20 20:22:21 2013
@@ -35,6 +35,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
@@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
@@ -196,22 +198,29 @@ public class TestContainerManager extend
InterruptedException {
containerManager.start();
- File scriptFile = new File(tmpDir, "scriptFile.sh");
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile =
new File(tmpDir, "start_file.txt").getAbsoluteFile();
- fileWriter.write("\numask 0"); // So that start file is readable by the test
- fileWriter.write("\necho Hello World! > " + processStartFile);
- fileWriter.write("\necho $$ >> " + processStartFile);
- fileWriter.write("\nexec sleep 100");
+
+ // ////// Construct the Container-id
+ ContainerId cId = createContainerId();
+
+ if (Shell.WINDOWS) {
+ fileWriter.println("@echo Hello World!> " + processStartFile);
+ fileWriter.println("@echo " + cId + ">> " + processStartFile);
+ fileWriter.println("@ping -n 100 127.0.0.1 >nul");
+ } else {
+ fileWriter.write("\numask 0"); // So that start file is readable by the test
+ fileWriter.write("\necho Hello World! > " + processStartFile);
+ fileWriter.write("\necho $$ >> " + processStartFile);
+ fileWriter.write("\nexec sleep 100");
+ }
fileWriter.close();
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- // ////// Construct the Container-id
- ContainerId cId = createContainerId();
-
containerLaunchContext.setUser(user);
URL resource_alpha =
@@ -230,14 +239,12 @@ public class TestContainerManager extend
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
- List<String> commands = new ArrayList<String>();
- commands.add("/bin/bash");
- commands.add(scriptFile.getAbsolutePath());
+ List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(100 * 1024 * 1024, 1));
+ BuilderUtils.newResource(100, 1)); // MB
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
@@ -264,12 +271,10 @@ public class TestContainerManager extend
// Assert that the process is alive
Assert.assertTrue("Process is not alive!",
- exec.signalContainer(user,
- pid, Signal.NULL));
+ DefaultContainerExecutor.containerIsAlive(pid));
// Once more
Assert.assertTrue("Process is not alive!",
- exec.signalContainer(user,
- pid, Signal.NULL));
+ DefaultContainerExecutor.containerIsAlive(pid));
StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
stopRequest.setContainerId(cId);
@@ -283,28 +288,39 @@ public class TestContainerManager extend
gcsRequest.setContainerId(cId);
ContainerStatus containerStatus =
containerManager.getContainerStatus(gcsRequest).getStatus();
- Assert.assertEquals(ExitCode.TERMINATED.getExitCode(),
- containerStatus.getExitStatus());
+ int expectedExitCode = Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
+ ExitCode.TERMINATED.getExitCode();
+ Assert.assertEquals(expectedExitCode, containerStatus.getExitStatus());
// Assert that the process is not alive anymore
Assert.assertFalse("Process is still alive!",
- exec.signalContainer(user,
- pid, Signal.NULL));
+ DefaultContainerExecutor.containerIsAlive(pid));
}
private void testContainerLaunchAndExit(int exitCode) throws IOException, InterruptedException {
- File scriptFile = new File(tmpDir, "scriptFile.sh");
+ File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
PrintWriter fileWriter = new PrintWriter(scriptFile);
File processStartFile =
new File(tmpDir, "start_file.txt").getAbsoluteFile();
- fileWriter.write("\numask 0"); // So that start file is readable by the test
- fileWriter.write("\necho Hello World! > " + processStartFile);
- fileWriter.write("\necho $$ >> " + processStartFile);
-
- // Have script throw an exit code at the end
- if (exitCode != 0) {
- fileWriter.write("\nexit "+exitCode);
+
+ // ////// Construct the Container-id
+ ContainerId cId = createContainerId();
+
+ if (Shell.WINDOWS) {
+ fileWriter.println("@echo Hello World!> " + processStartFile);
+ fileWriter.println("@echo " + cId + ">> " + processStartFile);
+ if (exitCode != 0) {
+ fileWriter.println("@exit " + exitCode);
+ }
+ } else {
+ fileWriter.write("\numask 0"); // So that start file is readable by the test
+ fileWriter.write("\necho Hello World! > " + processStartFile);
+ fileWriter.write("\necho $$ >> " + processStartFile);
+ // Have script throw an exit code at the end
+ if (exitCode != 0) {
+ fileWriter.write("\nexit "+exitCode);
+ }
}
fileWriter.close();
@@ -312,9 +328,6 @@ public class TestContainerManager extend
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- // ////// Construct the Container-id
- ContainerId cId = createContainerId();
-
containerLaunchContext.setUser(user);
URL resource_alpha =
@@ -333,14 +346,12 @@ public class TestContainerManager extend
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(containerLaunchContext.getUser());
- List<String> commands = new ArrayList<String>();
- commands.add("/bin/bash");
- commands.add(scriptFile.getAbsolutePath());
+ List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(100 * 1024 * 1024, 1));
+ BuilderUtils.newResource(100, 1)); // MB
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);