You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by bi...@apache.org on 2013/06/06 22:17:25 UTC
svn commit: r1490423 [1/2] - in
/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn:
hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn-api...
Author: bikas
Date: Thu Jun 6 20:17:23 2013
New Revision: 1490423
URL: http://svn.apache.org/r1490423
Log:
Merge r1490396 from branch-2 to branch-2.1-beta for YARN-750. Allow for black-listing resources in YARN API and Impl in CS (acmurthy via bikas)
Added:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceBlacklistRequest.java
- copied unchanged from r1490396, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceBlacklistRequest.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
- copied unchanged from r1490396, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceBlacklistRequestException.java
- copied unchanged from r1490396, hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceBlacklistRequestException.java
Modified:
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/AMRMProtocol.java Thu Jun 6 20:17:23 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* <p>The protocol between a live instance of <code>ApplicationMaster</code>
@@ -60,6 +61,8 @@ public interface AMRMProtocol {
* @return registration respose
* @throws YarnException
* @throws IOException
+ * @see RegisterApplicationMasterRequest
+ * @see RegisterApplicationMasterResponse
*/
public RegisterApplicationMasterResponse registerApplicationMaster(
RegisterApplicationMasterRequest request)
@@ -80,6 +83,8 @@ public interface AMRMProtocol {
* @return completion response
* @throws YarnException
* @throws IOException
+ * @see FinishApplicationMasterRequest
+ * @see FinishApplicationMasterResponse
*/
public FinishApplicationMasterResponse finishApplicationMaster(
FinishApplicationMasterRequest request)
@@ -91,12 +96,16 @@ public interface AMRMProtocol {
*
* <p>The <code>ApplicationMaster</code> uses this interface to provide a list
* of {@link ResourceRequest} and returns unused {@link Container} allocated
- * to it via {@link AllocateRequest}.</p>
+ * to it via {@link AllocateRequest}. Optionally, the
+ * <code>ApplicationMaster</code> can also <em>blacklist</em> resources
+ * which it doesn't want to use.</p>
*
* <p>This also doubles up as a <em>heartbeat</em> to let the
* <code>ResourceManager</code> know that the <code>ApplicationMaster</code>
* is alive. Thus, applications should periodically make this call to be kept
- * alive. The frequency depends on ??</p>
+ * alive. The frequency depends on
+ * {@link YarnConfiguration#RM_AM_EXPIRY_INTERVAL_MS} which defaults to
+ * {@link YarnConfiguration#DEFAULT_RM_AM_EXPIRY_INTERVAL_MS}.</p>
*
* <p>The <code>ResourceManager</code> responds with list of allocated
* {@link Container}, status of completed containers and headroom information
@@ -110,6 +119,8 @@ public interface AMRMProtocol {
* @return allocation response
* @throws YarnException
* @throws IOException
+ * @see AllocateRequest
+ * @see AllocateResponse
*/
public AllocateResponse allocate(AllocateRequest request)
throws YarnException, IOException;
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java Thu Jun 6 20:17:23 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -61,13 +62,15 @@ public abstract class AllocateRequest {
public static AllocateRequest newInstance(
ApplicationAttemptId applicationAttemptId, int responseID,
float appProgress, List<ResourceRequest> resourceAsk,
- List<ContainerId> containersToBeReleased) {
+ List<ContainerId> containersToBeReleased,
+ ResourceBlacklistRequest resourceBlacklistRequest) {
AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
allocateRequest.setApplicationAttemptId(applicationAttemptId);
allocateRequest.setResponseId(responseID);
allocateRequest.setProgress(appProgress);
allocateRequest.setAskList(resourceAsk);
allocateRequest.setReleaseList(containersToBeReleased);
+ allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
return allocateRequest;
}
@@ -127,6 +130,7 @@ public abstract class AllocateRequest {
* Get the list of <code>ResourceRequest</code> to update the
* <code>ResourceManager</code> about the application's resource requirements.
* @return the list of <code>ResourceRequest</code>
+ * @see ResourceRequest
*/
@Public
@Stable
@@ -138,6 +142,7 @@ public abstract class AllocateRequest {
* @param resourceRequests list of <code>ResourceRequest</code> to update the
* <code>ResourceManager</code> about the application's
* resource requirements
+ * @see ResourceRequest
*/
@Public
@Stable
@@ -157,10 +162,37 @@ public abstract class AllocateRequest {
* Set the list of <code>ContainerId</code> of containers being
* released by the <code>ApplicationMaster</code>
* @param releaseContainers list of <code>ContainerId</code> of
- * containers being released by the <
- * code>ApplicationMaster</code>
+ * containers being released by the
+ * <code>ApplicationMaster</code>
*/
@Public
@Stable
public abstract void setReleaseList(List<ContainerId> releaseContainers);
+
+ /**
+ * Get the <code>ResourceBlacklistRequest</code> being sent by the
+ * <code>ApplicationMaster</code>.
+ * @return the <code>ResourceBlacklistRequest</code> being sent by the
+ * <code>ApplicationMaster</code>
+ * @see ResourceBlacklistRequest
+ */
+ @Public
+ @Stable
+ public abstract ResourceBlacklistRequest getResourceBlacklistRequest();
+
+ /**
+ * Set the <code>ResourceBlacklistRequest</code> to inform the
+ * <code>ResourceManager</code> about the blacklist additions and removals
+ * per the <code>ApplicationMaster</code>.
+ *
+ * @param resourceBlacklistRequest the <code>ResourceBlacklistRequest</code>
+ * to inform the <code>ResourceManager</code> about
+ * the blacklist additions and removals
+ * per the <code>ApplicationMaster</code>
+ * @see ResourceBlacklistRequest
+ */
+ @Public
+ @Stable
+ public abstract void setResourceBlacklistRequest(
+ ResourceBlacklistRequest resourceBlacklistRequest);
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java Thu Jun 6 20:17:23 2013
@@ -25,12 +25,15 @@ import java.util.List;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
@@ -46,6 +49,7 @@ public class AllocateRequestPBImpl exten
private ApplicationAttemptId applicationAttemptID = null;
private List<ResourceRequest> ask = null;
private List<ContainerId> release = null;
+ private ResourceBlacklistRequest blacklistRequest = null;
public AllocateRequestPBImpl() {
@@ -94,6 +98,9 @@ public class AllocateRequestPBImpl exten
if (this.release != null) {
addReleasesToProto();
}
+ if (this.blacklistRequest != null) {
+ builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest));
+ }
}
private void mergeLocalToProto() {
@@ -161,6 +168,7 @@ public class AllocateRequestPBImpl exten
initAsks();
return this.ask;
}
+
@Override
public void setAskList(final List<ResourceRequest> resourceRequests) {
if(resourceRequests == null) {
@@ -171,6 +179,28 @@ public class AllocateRequestPBImpl exten
this.ask.addAll(resourceRequests);
}
+ @Override
+ public ResourceBlacklistRequest getResourceBlacklistRequest() {
+ AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.blacklistRequest != null) {
+ return this.blacklistRequest;
+ }
+ if (!p.hasBlacklistRequest()) {
+ return null;
+ }
+ this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest());
+ return this.blacklistRequest;
+ }
+
+ @Override
+ public void setResourceBlacklistRequest(ResourceBlacklistRequest blacklistRequest) {
+ maybeInitBuilder();
+ if (blacklistRequest == null) {
+ builder.clearBlacklistRequest();
+ }
+ this.blacklistRequest = blacklistRequest;
+ }
+
private void initAsks() {
if (this.ask != null) {
return;
@@ -302,4 +332,14 @@ public class AllocateRequestPBImpl exten
private ContainerIdProto convertToProtoFormat(ContainerId t) {
return ((ContainerIdPBImpl)t).getProto();
}
+
+ private ResourceBlacklistRequestPBImpl convertFromProtoFormat(ResourceBlacklistRequestProto p) {
+ return new ResourceBlacklistRequestPBImpl(p);
+ }
+
+ private ResourceBlacklistRequestProto convertToProtoFormat(ResourceBlacklistRequest t) {
+ return ((ResourceBlacklistRequestPBImpl)t).getProto();
+ }
+
+
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java Thu Jun 6 20:17:23 2013
@@ -40,9 +40,15 @@ import org.apache.hadoop.yarn.util.Recor
* </li>
* <li>{@link Resource} required for each request.</li>
* <li>
- * Number of containers of such specifications which are required
+ * Number of containers, of above specifications, which are required
* by the application.
* </li>
+ * <li>
+ * A boolean <em>relaxLocality</em> flag, defaulting to <code>true</code>,
+ * which tells the <code>ResourceManager</code> if the application wants
+ * locality to be loose (i.e. allows fall-through to rack or <em>any</em>)
+ * or strict (i.e. specify hard constraint on resource allocation).
+ * </li>
* </ul>
* </p>
*
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Thu Jun 6 20:17:23 2013
@@ -229,6 +229,11 @@ message PreemptionResourceRequestProto {
optional ResourceRequestProto resource = 1;
}
+message ResourceBlacklistRequestProto {
+ repeated string blacklist_additions = 1;
+ repeated string blacklist_removals = 2;
+}
+
////////////////////////////////////////////////////////////////////////
////// From client_RM_Protocol /////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Thu Jun 6 20:17:23 2013
@@ -54,8 +54,9 @@ message AllocateRequestProto {
optional ApplicationAttemptIdProto application_attempt_id = 1;
repeated ResourceRequestProto ask = 2;
repeated ContainerIdProto release = 3;
- optional int32 response_id = 4;
- optional float progress = 5;
+ optional ResourceBlacklistRequestProto blacklist_request = 4;
+ optional int32 response_id = 5;
+ optional float progress = 6;
}
message AllocateResponseProto {
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/AMRMClientImpl.java Thu Jun 6 20:17:23 2013
@@ -228,7 +228,7 @@ public class AMRMClientImpl<T extends Co
release.clear();
allocateRequest =
AllocateRequest.newInstance(appAttemptId, lastResponseId,
- progressIndicator, askList, releaseList);
+ progressIndicator, askList, releaseList, null);
}
allocateResponse = rmClient.allocate(allocateRequest);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Thu Jun 6 20:17:23 2013
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.PreemptionResourceRequest;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -69,6 +70,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -286,18 +288,35 @@ public class ApplicationMasterService ex
List<ResourceRequest> ask = request.getAskList();
List<ContainerId> release = request.getReleaseList();
-
+
+ ResourceBlacklistRequest blacklistRequest = request.getResourceBlacklistRequest();
+ List<String> blacklistAdditions =
+ (blacklistRequest != null) ?
+ blacklistRequest.getBlacklistAdditions() : null;
+ List<String> blacklistRemovals =
+ (blacklistRequest != null) ?
+ blacklistRequest.getBlacklistRemovals() : null;
+
// sanity check
try {
SchedulerUtils.validateResourceRequests(ask,
rScheduler.getMaximumResourceCapability());
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
- throw RPCUtil.getRemoteException(e);
+ throw e;
}
+
+ try {
+ SchedulerUtils.validateBlacklistRequest(blacklistRequest);
+ } catch (InvalidResourceBlacklistRequestException e) {
+ LOG.warn("Invalid blacklist request by application " + appAttemptId, e);
+ throw e;
+ }
+
// Send new requests to appAttempt.
Allocation allocation =
- this.rScheduler.allocate(appAttemptId, ask, release);
+ this.rScheduler.allocate(appAttemptId, ask, release,
+ blacklistAdditions, blacklistRemovals);
RMApp app = this.rmContext.getRMApps().get(
appAttemptId.getApplicationId());
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Thu Jun 6 20:17:23 2013
@@ -803,7 +803,7 @@ public class RMAppAttemptImpl implements
// AM resource has been checked when submission
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId,
- Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST);
+ Collections.singletonList(request), EMPTY_CONTAINER_RELEASE_LIST, null, null);
if (amContainerAllocation != null
&& amContainerAllocation.getContainers() != null) {
assert (amContainerAllocation.getContainers().size() == 0);
@@ -827,7 +827,7 @@ public class RMAppAttemptImpl implements
// Acquire the AM container from the scheduler.
Allocation amContainerAllocation = appAttempt.scheduler.allocate(
appAttempt.applicationAttemptId, EMPTY_CONTAINER_REQUEST_LIST,
- EMPTY_CONTAINER_RELEASE_LIST);
+ EMPTY_CONTAINER_RELEASE_LIST, null, null);
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java Thu Jun 6 20:17:23 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -59,7 +60,8 @@ public class AppSchedulingInfo {
new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator());
final Map<Priority, Map<String, ResourceRequest>> requests =
new HashMap<Priority, Map<String, ResourceRequest>>();
-
+ final Set<String> blacklist = new HashSet<String>();
+
//private final ApplicationStore store;
private final ActiveUsersManager activeUsersManager;
@@ -114,12 +116,15 @@ public class AppSchedulingInfo {
* application, by asking for more resources and releasing resources acquired
* by the application.
*
- * @param requests
- * resources to be acquired
+ * @param requests resources to be acquired
+ * @param blacklistAdditions resources to be added to the blacklist
+ * @param blacklistRemovals resources to be removed from the blacklist
*/
synchronized public void updateResourceRequests(
- List<ResourceRequest> requests) {
+ List<ResourceRequest> requests,
+ List<String> blacklistAdditions, List<String> blacklistRemovals) {
QueueMetrics metrics = queue.getMetrics();
+
// Update resource requests
for (ResourceRequest request : requests) {
Priority priority = request.getPriority();
@@ -175,6 +180,20 @@ public class AppSchedulingInfo {
lastRequestContainers)));
}
}
+
+ //
+ // Update blacklist
+ //
+
+ // Add to blacklist
+ if (blacklistAdditions != null) {
+ blacklist.addAll(blacklistAdditions);
+ }
+
+ // Remove from blacklist
+ if (blacklistRemovals != null) {
+ blacklist.removeAll(blacklistRemovals);
+ }
}
synchronized public Collection<Priority> getPriorities() {
@@ -197,6 +216,10 @@ public class AppSchedulingInfo {
return request.getCapability();
}
+ public synchronized boolean isBlacklisted(String resourceName) {
+ return blacklist.contains(resourceName);
+ }
+
/**
* Resources have been allocated to this application by the resource
* scheduler. Track them.
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/InvalidResourceRequestException.java Thu Jun 6 20:17:23 2013
@@ -18,14 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-import org.apache.hadoop.yarn.YarnRuntimeException;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.exceptions.YarnException;
/**
- * The exception is thrown when the requested resource is out of the range
+ * The exception is thrown when a {@link ResourceRequest} is out of the range
* of the configured lower and upper resource boundaries.
*
*/
-public class InvalidResourceRequestException extends YarnRuntimeException {
+public class InvalidResourceRequestException extends YarnException {
+
+ private static final long serialVersionUID = 13498237L;
public InvalidResourceRequestException(Throwable cause) {
super(cause);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Thu Jun 6 20:17:23 2013
@@ -22,6 +22,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -152,4 +153,14 @@ public class SchedulerUtils {
}
}
+ public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest)
+ throws InvalidResourceBlacklistRequestException {
+ if (blacklistRequest != null) {
+ List<String> plus = blacklistRequest.getBlacklistAdditions();
+ if (plus != null && plus.contains(ResourceRequest.ANY)) {
+ throw new InvalidResourceBlacklistRequestException(
+ "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
+ }
+ }
+ }
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java Thu Jun 6 20:17:23 2013
@@ -95,6 +95,8 @@ public interface YarnScheduler extends E
* @param appAttemptId
* @param ask
* @param release
+ * @param blacklistAdditions
+ * @param blacklistRemovals
* @return the {@link Allocation} for the application
*/
@Public
@@ -102,7 +104,9 @@ public interface YarnScheduler extends E
Allocation
allocate(ApplicationAttemptId appAttemptId,
List<ResourceRequest> ask,
- List<ContainerId> release);
+ List<ContainerId> release,
+ List<String> blacklistAdditions,
+ List<String> blacklistRemovals);
/**
* Get node resource usage report.
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Thu Jun 6 20:17:23 2013
@@ -472,7 +472,8 @@ implements ResourceScheduler, CapacitySc
@Override
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
- List<ResourceRequest> ask, List<ContainerId> release) {
+ List<ResourceRequest> ask, List<ContainerId> release,
+ List<String> blacklistAdditions, List<String> blacklistRemovals) {
FiCaSchedulerApp application = getApplication(applicationAttemptId);
if (application == null) {
@@ -523,7 +524,8 @@ implements ResourceScheduler, CapacitySc
application.showRequests();
// Update application requests
- application.updateResourceRequests(ask);
+ application.updateResourceRequests(ask,
+ blacklistAdditions, blacklistRemovals);
LOG.debug("allocate: post-update");
application.showRequests();
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Jun 6 20:17:23 2013
@@ -815,6 +815,11 @@ public class LeafQueue implements CSQueu
}
synchronized (application) {
+ // Check if this resource is on the blacklist
+ if (isBlacklisted(application, node)) {
+ continue;
+ }
+
// Schedule in priority order
for (Priority priority : application.getPriorities()) {
// Required resource
@@ -897,6 +902,28 @@ public class LeafQueue implements CSQueu
return NULL_ASSIGNMENT;
}
+
+ boolean isBlacklisted(FiCaSchedulerApp application, FiCaSchedulerNode node) {
+ if (application.isBlacklisted(node.getHostName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping 'host' " + node.getHostName() +
+ " for " + application.getApplicationId() +
+ " since it has been blacklisted");
+ }
+ return true;
+ }
+
+ if (application.isBlacklisted(node.getRackName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skipping 'rack' " + node.getRackName() +
+ " for " + application.getApplicationId() +
+ " since it has been blacklisted");
+ }
+ return true;
+ }
+
+ return false;
+ }
private synchronized CSAssignment
assignReservedContainer(FiCaSchedulerApp application,
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java Thu Jun 6 20:17:23 2013
@@ -134,9 +134,11 @@ public class FiCaSchedulerApp extends Sc
}
public synchronized void updateResourceRequests(
- List<ResourceRequest> requests) {
+ List<ResourceRequest> requests,
+ List<String> blacklistAdditions, List<String> blacklistRemovals) {
if (!isStopped) {
- this.appSchedulingInfo.updateResourceRequests(requests);
+ this.appSchedulingInfo.updateResourceRequests(requests,
+ blacklistAdditions, blacklistRemovals);
}
}
@@ -163,6 +165,10 @@ public class FiCaSchedulerApp extends Sc
public Resource getResource(Priority priority) {
return this.appSchedulingInfo.getResource(priority);
}
+
+ public boolean isBlacklisted(String resourceName) {
+ return this.appSchedulingInfo.isBlacklisted(resourceName);
+ }
/**
* Is this application pending?
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java Thu Jun 6 20:17:23 2013
@@ -138,7 +138,7 @@ public class FSSchedulerApp extends Sche
public synchronized void updateResourceRequests(
List<ResourceRequest> requests) {
- this.appSchedulingInfo.updateResourceRequests(requests);
+ this.appSchedulingInfo.updateResourceRequests(requests, null, null);
}
public Map<String, ResourceRequest> getResourceRequests(Priority priority) {
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu Jun 6 20:17:23 2013
@@ -718,7 +718,7 @@ public class FairScheduler implements Re
@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
- List<ResourceRequest> ask, List<ContainerId> release) {
+ List<ResourceRequest> ask, List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
// Make sure this application exists
FSSchedulerApp application = applications.get(appAttemptId);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Thu Jun 6 20:17:23 2013
@@ -222,7 +222,7 @@ public class FifoScheduler implements Re
@Override
public Allocation allocate(
ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
- List<ContainerId> release) {
+ List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
FiCaSchedulerApp application = getApplication(applicationAttemptId);
if (application == null) {
LOG.error("Calling allocate on removed " +
@@ -268,7 +268,7 @@ public class FifoScheduler implements Re
application.showRequests();
// Update application requests
- application.updateResourceRequests(ask);
+ application.updateResourceRequests(ask, null, null);
LOG.debug("allocate: post-update" +
" applicationId=" + applicationAttemptId +
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Thu Jun 6 20:17:23 2013
@@ -265,7 +265,7 @@ public class Application {
// Get resources from the ResourceManager
resourceManager.getResourceScheduler().allocate(applicationAttemptId,
- new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>());
+ new ArrayList<ResourceRequest>(ask), new ArrayList<ContainerId>(), null, null);
System.out.println("-=======" + applicationAttemptId);
System.out.println("----------" + resourceManager.getRMContext().getRMApps()
.get(applicationId).getRMAppAttempt(applicationAttemptId));
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Thu Jun 6 20:17:23 2013
@@ -144,7 +144,7 @@ public class MockAM {
List<ResourceRequest> resourceRequest, List<ContainerId> releases)
throws Exception {
AllocateRequest req = AllocateRequest.newInstance(attemptId,
- ++responseId, 0F, resourceRequest, releases);
+ ++responseId, 0F, resourceRequest, releases, null);
return amRMProtocol.allocate(req);
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Thu Jun 6 20:17:23 2013
@@ -273,23 +273,23 @@ public class TestFifoScheduler {
List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
- fs.allocate(appAttemptId1, ask1, emptyId);
+ fs.allocate(appAttemptId1, ask1, emptyId, null, null);
// Ask for a 2 GB container for app 2
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
- fs.allocate(appAttemptId2, ask2, emptyId);
+ fs.allocate(appAttemptId2, ask2, emptyId, null, null);
// Trigger container assignment
fs.handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
- Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId);
+ Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
Assert.assertEquals("Allocation headroom", 1 * GB,
allocation1.getResourceLimit().getMemory());
- Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId);
+ Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
Assert.assertEquals("Allocation headroom", 1 * GB,
allocation2.getResourceLimit().getMemory());
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java Thu Jun 6 20:17:23 2013
@@ -108,7 +108,7 @@ public class TestAMRMRPCNodeUpdates {
// allocate request returns no updated node
AllocateRequest allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), 0, 0F, null, null);
+ .getAppAttemptId(), 0, 0F, null, null, null);
AllocateResponse response1 = amService.allocate(allocateRequest1);
List<NodeReport> updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
@@ -117,7 +117,7 @@ public class TestAMRMRPCNodeUpdates {
// allocate request returns updated node
allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), response1.getResponseId(), 0F, null, null);
+ .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null);
response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
@@ -137,7 +137,7 @@ public class TestAMRMRPCNodeUpdates {
// subsequent allocate request returns delta
allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), response1.getResponseId(), 0F, null, null);
+ .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null);
response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
@@ -157,7 +157,7 @@ public class TestAMRMRPCNodeUpdates {
// allocate request returns no updated node
AllocateRequest allocateRequest2 = AllocateRequest.newInstance(attempt2
- .getAppAttemptId(), 0, 0F, null, null);
+ .getAppAttemptId(), 0, 0F, null, null, null);
AllocateResponse response2 = amService.allocate(allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
@@ -166,7 +166,7 @@ public class TestAMRMRPCNodeUpdates {
// both AM's should get delta updated nodes
allocateRequest1 = AllocateRequest.newInstance(attempt1
- .getAppAttemptId(), response1.getResponseId(), 0F, null, null);
+ .getAppAttemptId(), response1.getResponseId(), 0F, null, null, null);
response1 = amService.allocate(allocateRequest1);
updatedNodes = response1.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
@@ -175,7 +175,7 @@ public class TestAMRMRPCNodeUpdates {
Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
allocateRequest2 = AllocateRequest.newInstance(attempt2
- .getAppAttemptId(), response2.getResponseId(), 0F, null, null);
+ .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null);
response2 = amService.allocate(allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(1, updatedNodes.size());
@@ -185,7 +185,7 @@ public class TestAMRMRPCNodeUpdates {
// subsequent allocate calls should return no updated nodes
allocateRequest2 = AllocateRequest.newInstance(attempt2
- .getAppAttemptId(), response2.getResponseId(), 0F, null, null);
+ .getAppAttemptId(), response2.getResponseId(), 0F, null, null, null);
response2 = amService.allocate(allocateRequest2);
updatedNodes = response2.getUpdatedNodes();
Assert.assertEquals(0, updatedNodes.size());
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java Thu Jun 6 20:17:23 2013
@@ -78,13 +78,13 @@ public class TestAMRMRPCResponseId {
am.registerAppAttempt();
AllocateRequest allocateRequest = AllocateRequest.newInstance(attempt
- .getAppAttemptId(), 0, 0F, null, null);
+ .getAppAttemptId(), 0, 0F, null, null, null);
AllocateResponse response = amService.allocate(allocateRequest);
Assert.assertEquals(1, response.getResponseId());
Assert.assertFalse(response.getResync());
allocateRequest = AllocateRequest.newInstance(attempt
- .getAppAttemptId(), response.getResponseId(), 0F, null, null);
+ .getAppAttemptId(), response.getResponseId(), 0F, null, null, null);
response = amService.allocate(allocateRequest);
Assert.assertEquals(2, response.getResponseId());
@@ -94,7 +94,7 @@ public class TestAMRMRPCResponseId {
/** try sending old request again **/
allocateRequest = AllocateRequest.newInstance(attempt
- .getAppAttemptId(), 0, 0F, null, null);
+ .getAppAttemptId(), 0, 0F, null, null, null);
response = amService.allocate(allocateRequest);
Assert.assertTrue(response.getResync());
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Jun 6 20:17:23 2013
@@ -330,7 +330,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptState());
verify(scheduler, times(expectedAllocateCount)).
allocate(any(ApplicationAttemptId.class),
- any(List.class), any(List.class));
+ any(List.class), any(List.class), any(List.class), any(List.class));
assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
assertNull(applicationAttempt.getMasterContainer());
@@ -345,6 +345,7 @@ public class TestRMAppAttemptTransitions
/**
* {@link RMAppAttemptState#ALLOCATED}
*/
+ @SuppressWarnings("unchecked")
private void testAppAttemptAllocatedState(Container amContainer) {
assertEquals(RMAppAttemptState.ALLOCATED,
applicationAttempt.getAppAttemptState());
@@ -354,7 +355,9 @@ public class TestRMAppAttemptTransitions
verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class));
verify(scheduler, times(2)).
allocate(
- any(ApplicationAttemptId.class), any(List.class), any(List.class));
+ any(
+ ApplicationAttemptId.class), any(List.class), any(List.class),
+ any(List.class), any(List.class));
}
/**
@@ -465,6 +468,7 @@ public class TestRMAppAttemptTransitions
testAppAttemptScheduledState();
}
+ @SuppressWarnings("unchecked")
private Container allocateApplicationAttempt() {
scheduleApplicationAttempt();
@@ -481,6 +485,8 @@ public class TestRMAppAttemptTransitions
scheduler.allocate(
any(ApplicationAttemptId.class),
any(List.class),
+ any(List.class),
+ any(List.class),
any(List.class))).
thenReturn(allocation);
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Thu Jun 6 20:17:23 2013
@@ -22,20 +22,47 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
+import java.security.PrivilegedAction;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
+import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.Assert;
import org.junit.Test;
public class TestSchedulerUtils {
+ private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class);
+
@Test (timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -239,5 +266,80 @@ public class TestSchedulerUtils {
// expected
}
}
+
+ @Test
+ public void testValidateResourceBlacklistRequest() throws Exception {
+
+ MyContainerManager containerManager = new MyContainerManager();
+ final MockRM rm =
+ new MockRMWithAMS(new YarnConfiguration(), containerManager);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("localhost:1234", 5120);
+
+ Map<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>(2);
+ acls.put(ApplicationAccessType.VIEW_APP, "*");
+ RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
+
+ nm1.nodeHeartbeat(true);
+
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
+ waitForLaunchedState(attempt);
+
+ // Create a client to the RM.
+ final Configuration conf = rm.getConfig();
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ UserGroupInformation currentUser =
+ UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
+
+ AMRMProtocol client = currentUser
+ .doAs(new PrivilegedAction<AMRMProtocol>() {
+ @Override
+ public AMRMProtocol run() {
+ return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rm
+ .getApplicationMasterService().getBindAddress(), conf);
+ }
+ });
+
+ RegisterApplicationMasterRequest request = Records
+ .newRecord(RegisterApplicationMasterRequest.class);
+ request.setApplicationAttemptId(applicationAttemptId);
+ client.registerApplicationMaster(request);
+
+ ResourceBlacklistRequest blacklistRequest =
+ ResourceBlacklistRequest.newInstance(
+ Collections.singletonList(ResourceRequest.ANY), null);
+
+ AllocateRequest allocateRequest =
+ AllocateRequest.newInstance(applicationAttemptId, 0, 0.0f, null, null,
+ blacklistRequest);
+ boolean error = false;
+ try {
+ client.allocate(allocateRequest);
+ } catch (InvalidResourceBlacklistRequestException e) {
+ error = true;
+ }
+
+ rm.stop();
+
+ Assert.assertTrue(
+ "Didn't not catch InvalidResourceBlacklistRequestException", error);
+ }
+
+ private void waitForLaunchedState(RMAppAttempt attempt)
+ throws InterruptedException {
+ int waitCount = 0;
+ while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED
+ && waitCount++ < 20) {
+ LOG.info("Waiting for AppAttempt to reach LAUNCHED state. "
+ + "Current state is " + attempt.getAppAttemptState());
+ Thread.sleep(1000);
+ }
+ Assert.assertEquals(attempt.getAppAttemptState(),
+ RMAppAttemptState.LAUNCHED);
+ }
}
Modified: hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java?rev=1490423&r1=1490422&r2=1490423&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java (original)
+++ hadoop/common/branches/branch-2.1-beta/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java Thu Jun 6 20:17:23 2013
@@ -513,7 +513,7 @@ public class TestApplicationLimits {
app_0_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_0.updateResourceRequests(app_0_0_requests);
+ app_0_0.updateResourceRequests(app_0_0_requests, null, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0);
@@ -532,7 +532,7 @@ public class TestApplicationLimits {
app_0_1_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_0_1.updateResourceRequests(app_0_1_requests);
+ app_0_1.updateResourceRequests(app_0_1_requests, null, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute
@@ -551,7 +551,7 @@ public class TestApplicationLimits {
app_1_0_requests.add(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2,
true, priority_1, recordFactory));
- app_1_0.updateResourceRequests(app_1_0_requests);
+ app_1_0.updateResourceRequests(app_1_0_requests, null, null);
// Schedule to compute
queue.assignContainers(clusterResource, node_0); // Schedule to compute