You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2018/01/17 21:58:55 UTC
[01/11] hadoop git commit: YARN-7670. Modifications to the
ResourceScheduler API to support SchedulingRequests. (asuresh) [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-6592 97141f1e0 -> 4582e7b38 (forced update)
YARN-7670. Modifications to the ResourceScheduler API to support SchedulingRequests. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39676318
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39676318
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39676318
Branch: refs/heads/YARN-6592
Commit: 396763189b7b0f1b9d219e2c12781615a664bda9
Parents: 3f037a1
Author: Arun Suresh <as...@apache.org>
Authored: Tue Dec 19 08:59:23 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:44:20 2018 -0800
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 18 +++++
.../scheduler/ResourceScheduler.java | 13 ++++
.../scheduler/capacity/CapacityScheduler.java | 78 ++++++++++++++++++--
.../common/ResourceAllocationCommitter.java | 12 ++-
.../scheduler/common/fica/FiCaSchedulerApp.java | 30 +++++---
.../TestCapacitySchedulerAsyncScheduling.java | 10 +--
6 files changed, 138 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39676318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index cf5e13b..1589d84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -296,6 +297,10 @@ public abstract class AbstractYarnScheduler
return nodeTracker.getNodes(nodeFilter);
}
+ public List<N> getNodes(final NodeFilter filter) {
+ return nodeTracker.getNodes(filter);
+ }
+
public boolean shouldContainersBeAutoUpdated() {
return this.autoUpdateContainers;
}
@@ -1439,4 +1444,17 @@ public abstract class AbstractYarnScheduler
throw new IOException(e);
}
}
+
+ /**
+ * Default implementation. Always returns false.
+ * @param appAttempt ApplicationAttempt.
+ * @param schedulingRequest SchedulingRequest.
+ * @param schedulerNode SchedulerNode.
+ * @return Success or not.
+ */
+ @Override
+ public boolean attemptAllocationOnNode(SchedulerApplicationAttempt appAttempt,
+ SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39676318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
index d96d625..5a56ac7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -58,4 +59,16 @@ public interface ResourceScheduler extends YarnScheduler, Recoverable {
* @return the number of available {@link NodeId} by resource name.
*/
List<NodeId> getNodeIds(String resourceName);
+
+ /**
+ * Attempts to allocate a SchedulerRequest on a Node.
+ * NOTE: This ignores the numAllocations in the resource sizing and tries
+ * to allocate a SINGLE container only.
+ * @param appAttempt ApplicationAttempt.
+ * @param schedulingRequest SchedulingRequest.
+ * @param schedulerNode SchedulerNode.
+ * @return true if proposal was accepted.
+ */
+ boolean attemptAllocationOnNode(SchedulerApplicationAttempt appAttempt,
+ SchedulingRequest schedulingRequest, SchedulerNode schedulerNode);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39676318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 000f59c..d92ce58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -81,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
@@ -98,7 +101,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesLogger;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
@@ -140,6 +145,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.Candida
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimpleCandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -551,7 +558,7 @@ public class CapacityScheduler extends
try {
cs.writeLock.lock();
- cs.tryCommit(cs.getClusterResource(), request);
+ cs.tryCommit(cs.getClusterResource(), request, true);
} finally {
cs.writeLock.unlock();
}
@@ -2489,10 +2496,67 @@ public class CapacityScheduler extends
resourceCommitterService.addNewCommitRequest(request);
} else{
// Otherwise do it sync-ly.
- tryCommit(cluster, request);
+ tryCommit(cluster, request, true);
}
}
+ @Override
+ public boolean attemptAllocationOnNode(SchedulerApplicationAttempt appAttempt,
+ SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
+ if (schedulingRequest.getResourceSizing() != null) {
+ if (schedulingRequest.getResourceSizing().getNumAllocations() > 1) {
+ LOG.warn("The SchedulingRequest has requested more than 1 allocation," +
+ " but only 1 will be attempted !!");
+ }
+ if (!appAttempt.isStopped()) {
+ ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
+ resourceCommitRequest = createResourceCommitRequest(
+ appAttempt, schedulingRequest, schedulerNode);
+ return tryCommit(getClusterResource(), resourceCommitRequest, false);
+ }
+ }
+ return false;
+ }
+
+ // This assumes numContainers = 1 for the request.
+ private ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
+ createResourceCommitRequest(SchedulerApplicationAttempt appAttempt,
+ SchedulingRequest schedulingRequest, SchedulerNode schedulerNode) {
+ ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode> allocated =
+ null;
+ Resource resource = schedulingRequest.getResourceSizing().getResources();
+ if (Resources.greaterThan(calculator, getClusterResource(),
+ resource, Resources.none())) {
+ ContainerId cId =
+ ContainerId.newContainerId(appAttempt.getApplicationAttemptId(),
+ appAttempt.getAppSchedulingInfo().getNewContainerId());
+ Container container = BuilderUtils.newContainer(
+ cId, schedulerNode.getNodeID(), schedulerNode.getHttpAddress(),
+ resource, schedulingRequest.getPriority(), null,
+ ExecutionType.GUARANTEED,
+ schedulingRequest.getAllocationRequestId());
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.extractFrom(container),
+ appAttempt.getApplicationAttemptId(), container.getNodeId(),
+ appAttempt.getUser(), rmContext, false);
+
+ allocated = new ContainerAllocationProposal<>(
+ getSchedulerContainer(rmContainer, true),
+ null, null, NodeType.NODE_LOCAL, NodeType.NODE_LOCAL,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY,
+ resource);
+ }
+
+ if (null != allocated) {
+ List<ContainerAllocationProposal<FiCaSchedulerApp, FiCaSchedulerNode>>
+ allocationsList = new ArrayList<>();
+ allocationsList.add(allocated);
+
+ return new ResourceCommitRequest<>(allocationsList, null, null);
+ }
+ return null;
+ }
+
@VisibleForTesting
public ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
createResourceCommitRequest(CSAssignment csAssignment) {
@@ -2570,7 +2634,8 @@ public class CapacityScheduler extends
}
@Override
- public void tryCommit(Resource cluster, ResourceCommitRequest r) {
+ public boolean tryCommit(Resource cluster, ResourceCommitRequest r,
+ boolean updatePending) {
ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request =
(ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>) r;
@@ -2600,15 +2665,17 @@ public class CapacityScheduler extends
LOG.debug("Try to commit allocation proposal=" + request);
}
+ boolean isSuccess = false;
if (attemptId != null) {
FiCaSchedulerApp app = getApplicationAttempt(attemptId);
// Required sanity check for attemptId - when async-scheduling enabled,
// proposal might be outdated if AM failover just finished
// and proposal queue was not be consumed in time
if (app != null && attemptId.equals(app.getApplicationAttemptId())) {
- if (app.accept(cluster, request)) {
- app.apply(cluster, request);
+ if (app.accept(cluster, request, updatePending)) {
+ app.apply(cluster, request, updatePending);
LOG.info("Allocation proposal accepted");
+ isSuccess = true;
} else{
LOG.info("Failed to accept allocation proposal");
}
@@ -2619,6 +2686,7 @@ public class CapacityScheduler extends
}
}
}
+ return isSuccess;
}
public int getAsyncSchedulingPendingBacklogs() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39676318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ResourceAllocationCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ResourceAllocationCommitter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ResourceAllocationCommitter.java
index bdea97d..2e36b2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ResourceAllocationCommitter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/ResourceAllocationCommitter.java
@@ -25,5 +25,15 @@ import org.apache.hadoop.yarn.api.records.Resource;
* plus global scheduling functionality
*/
public interface ResourceAllocationCommitter {
- void tryCommit(Resource cluster, ResourceCommitRequest proposal);
+
+ /**
+ * Try to commit the allocation Proposal. This also gives the option of
+ * not updating a pending queued request.
+ * @param cluster Cluster Resource.
+ * @param proposal Proposal.
+ * @param updatePending Decrement pending if successful.
+ * @return Is successful or not.
+ */
+ boolean tryCommit(Resource cluster, ResourceCommitRequest proposal,
+ boolean updatePending);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39676318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index d6ad292..4ea0347 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -375,7 +375,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
}
public boolean accept(Resource cluster,
- ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
+ ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request,
+ boolean checkPending) {
ContainerRequest containerRequest = null;
boolean reReservation = false;
@@ -408,9 +409,11 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
schedulerContainer.getRmContainer().getContainerRequest();
// Check pending resource request
- if (!appSchedulingInfo.checkAllocation(allocation.getAllocationLocalityType(),
- schedulerContainer.getSchedulerNode(),
- schedulerContainer.getSchedulerRequestKey())) {
+ if (checkPending &&
+ !appSchedulingInfo.checkAllocation(
+ allocation.getAllocationLocalityType(),
+ schedulerContainer.getSchedulerNode(),
+ schedulerContainer.getSchedulerRequestKey())) {
if (LOG.isDebugEnabled()) {
LOG.debug("No pending resource for: nodeType=" + allocation
.getAllocationLocalityType() + ", node=" + schedulerContainer
@@ -485,8 +488,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return accepted;
}
- public void apply(Resource cluster,
- ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode> request) {
+ public void apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,
+ FiCaSchedulerNode> request, boolean updatePending) {
boolean reReservation = false;
try {
@@ -531,12 +534,15 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
liveContainers.put(containerId, rmContainer);
// Deduct pending resource requests
- ContainerRequest containerRequest = appSchedulingInfo.allocate(
- allocation.getAllocationLocalityType(),
- schedulerContainer.getSchedulerNode(),
- schedulerContainer.getSchedulerRequestKey(),
- schedulerContainer.getRmContainer().getContainer());
- ((RMContainerImpl) rmContainer).setContainerRequest(containerRequest);
+ if (updatePending) {
+ ContainerRequest containerRequest = appSchedulingInfo.allocate(
+ allocation.getAllocationLocalityType(),
+ schedulerContainer.getSchedulerNode(),
+ schedulerContainer.getSchedulerRequestKey(),
+ schedulerContainer.getRmContainer().getContainer());
+ ((RMContainerImpl) rmContainer).setContainerRequest(
+ containerRequest);
+ }
attemptResourceUsage.incUsed(schedulerContainer.getNodePartition(),
allocation.getAllocatedOrReservedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39676318/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 77596e2..6cb21d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -259,7 +259,7 @@ public class TestCapacitySchedulerAsyncScheduling {
reservedProposals.add(reservedForAttempt1Proposal);
ResourceCommitRequest request =
new ResourceCommitRequest(null, reservedProposals, null);
- scheduler.tryCommit(scheduler.getClusterResource(), request);
+ scheduler.tryCommit(scheduler.getClusterResource(), request, true);
Assert.assertNull("Outdated proposal should not be accepted!",
sn2.getReservedContainer());
@@ -380,7 +380,7 @@ public class TestCapacitySchedulerAsyncScheduling {
// call real apply
try {
cs.tryCommit((Resource) invocation.getArguments()[0],
- (ResourceCommitRequest) invocation.getArguments()[1]);
+ (ResourceCommitRequest) invocation.getArguments()[1], true);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
@@ -388,12 +388,12 @@ public class TestCapacitySchedulerAsyncScheduling {
isChecked.set(true);
} else {
cs.tryCommit((Resource) invocation.getArguments()[0],
- (ResourceCommitRequest) invocation.getArguments()[1]);
+ (ResourceCommitRequest) invocation.getArguments()[1], true);
}
return null;
}
}).when(spyCs).tryCommit(Mockito.any(Resource.class),
- Mockito.any(ResourceCommitRequest.class));
+ Mockito.any(ResourceCommitRequest.class), Mockito.anyBoolean());
spyCs.handle(new NodeUpdateSchedulerEvent(sn1.getRMNode()));
@@ -468,7 +468,7 @@ public class TestCapacitySchedulerAsyncScheduling {
newProposals.add(newContainerProposal);
ResourceCommitRequest request =
new ResourceCommitRequest(newProposals, null, null);
- scheduler.tryCommit(scheduler.getClusterResource(), request);
+ scheduler.tryCommit(scheduler.getClusterResource(), request, true);
}
// make sure node resource can't be over-allocated!
Assert.assertTrue("Node resource is Over-allocated!",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/11] hadoop git commit: YARN-7696. Add container tags to
ContainerTokenIdentifier,
api.Container and NMContainerStatus to handle all recovery cases. (asuresh)
Posted by as...@apache.org.
YARN-7696. Add container tags to ContainerTokenIdentifier, api.Container and NMContainerStatus to handle all recovery cases. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/308fc39e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/308fc39e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/308fc39e
Branch: refs/heads/YARN-6592
Commit: 308fc39e3b9d15595aac7bfdb2d29fa90e823594
Parents: 3f57235
Author: Arun Suresh <as...@apache.org>
Authored: Fri Jan 12 14:37:06 2018 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:53:24 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/Container.java | 15 +++++
.../src/main/proto/yarn_protos.proto | 1 +
.../api/records/impl/pb/ContainerPBImpl.java | 31 +++++++++
.../yarn/security/ContainerTokenIdentifier.java | 69 +++++++++++++++++++-
.../src/main/proto/yarn_security_token.proto | 1 +
.../api/protocolrecords/NMContainerStatus.java | 14 ++++
.../impl/pb/NMContainerStatusPBImpl.java | 33 ++++++++++
.../yarn_server_common_service_protos.proto | 1 +
.../containermanager/ContainerManagerImpl.java | 3 +-
.../container/ContainerImpl.java | 19 +++---
.../rmcontainer/RMContainerImpl.java | 10 ++-
.../scheduler/SchedulerApplicationAttempt.java | 3 +-
.../security/RMContainerTokenSecretManager.java | 21 ++----
.../capacity/TestContainerAllocation.java | 5 +-
14 files changed, 194 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
index 4fdc803..b9ca3f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.util.Records;
+import java.util.Collections;
+import java.util.Set;
+
/**
* {@code Container} represents an allocated resource in the cluster.
* <p>
@@ -256,4 +259,16 @@ public abstract class Container implements Comparable<Container> {
public void setVersion(int version) {
throw new UnsupportedOperationException();
}
+
+ @Private
+ @Unstable
+ public Set<String> getAllocationTags() {
+ return Collections.EMPTY_SET;
+ }
+
+ @Private
+ @Unstable
+ public void setAllocationTags(Set<String> allocationTags) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5cb1177..25c8569 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -152,6 +152,7 @@ message ContainerProto {
optional ExecutionTypeProto execution_type = 7 [default = GUARANTEED];
optional int64 allocation_request_id = 8 [default = -1];
optional int32 version = 9 [default = 0];
+ repeated string allocation_tags = 10;
}
message ContainerReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
index be84938..47be2f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
@@ -36,6 +36,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto;
+import java.util.HashSet;
+import java.util.Set;
+
@Private
@Unstable
public class ContainerPBImpl extends Container {
@@ -49,6 +52,7 @@ public class ContainerPBImpl extends Container {
private Resource resource = null;
private Priority priority = null;
private Token containerToken = null;
+ private Set<String> allocationTags = null;
public ContainerPBImpl() {
builder = ContainerProto.newBuilder();
@@ -106,6 +110,10 @@ public class ContainerPBImpl extends Container {
builder.getContainerToken())) {
builder.setContainerToken(convertToProtoFormat(this.containerToken));
}
+ if (this.allocationTags != null) {
+ builder.clearAllocationTags();
+ builder.addAllAllocationTags(this.allocationTags);
+ }
}
private void mergeLocalToProto() {
@@ -284,6 +292,29 @@ public class ContainerPBImpl extends Container {
builder.setVersion(version);
}
+ private void initAllocationTags() {
+ if (this.allocationTags != null) {
+ return;
+ }
+ ContainerProtoOrBuilder p = viaProto ? proto : builder;
+ this.allocationTags = new HashSet<>();
+ this.allocationTags.addAll(p.getAllocationTagsList());
+ }
+
+ @Override
+ public Set<String> getAllocationTags() {
+ initAllocationTags();
+ return this.allocationTags;
+ }
+
+ @Override
+ public void setAllocationTags(Set<String> allocationTags) {
+ maybeInitBuilder();
+ builder.clearAllocationTags();
+ this.allocationTags = allocationTags;
+ }
+
+
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
return new ContainerIdPBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 9e7d132..70935cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -22,6 +22,9 @@ import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -115,7 +118,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
this(containerID, 0, hostName, appSubmitter, r, expiryTimeStamp,
masterKeyId, rmIdentifier, priority, creationTime,
logAggregationContext, nodeLabelExpression, containerType,
- ExecutionType.GUARANTEED, -1);
+ ExecutionType.GUARANTEED, -1, null);
}
public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
@@ -127,15 +130,66 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
this(containerID, containerVersion, hostName, appSubmitter, r,
expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime,
logAggregationContext, nodeLabelExpression, containerType,
- executionType, -1);
+ executionType, -1, null);
}
+ /**
+ * Convenience Constructor for existing clients.
+ *
+ * @param containerID containerID
+ * @param containerVersion containerVersion
+ * @param hostName hostName
+ * @param appSubmitter appSubmitter
+ * @param r resource
+ * @param expiryTimeStamp expiryTimeStamp
+ * @param masterKeyId masterKeyId
+ * @param rmIdentifier rmIdentifier
+ * @param priority priority
+ * @param creationTime creationTime
+ * @param logAggregationContext logAggregationContext
+ * @param nodeLabelExpression nodeLabelExpression
+ * @param containerType containerType
+ * @param executionType executionType
+ * @param allocationRequestId allocationRequestId
+ */
public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
ContainerType containerType, ExecutionType executionType,
long allocationRequestId) {
+ this(containerID, containerVersion, hostName, appSubmitter, r,
+ expiryTimeStamp, masterKeyId, rmIdentifier, priority, creationTime,
+ logAggregationContext, nodeLabelExpression, containerType,
+ executionType, allocationRequestId, null);
+ }
+
+ /**
+ * Create a Container Token Identifier.
+ *
+ * @param containerID containerID
+ * @param containerVersion containerVersion
+ * @param hostName hostName
+ * @param appSubmitter appSubmitter
+ * @param r resource
+ * @param expiryTimeStamp expiryTimeStamp
+ * @param masterKeyId masterKeyId
+ * @param rmIdentifier rmIdentifier
+ * @param priority priority
+ * @param creationTime creationTime
+ * @param logAggregationContext logAggregationContext
+ * @param nodeLabelExpression nodeLabelExpression
+ * @param containerType containerType
+ * @param executionType executionType
+ * @param allocationRequestId allocationRequestId
+ * @param allocationTags Set of allocation Tags.
+ */
+ public ContainerTokenIdentifier(ContainerId containerID, int containerVersion,
+ String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
+ int masterKeyId, long rmIdentifier, Priority priority, long creationTime,
+ LogAggregationContext logAggregationContext, String nodeLabelExpression,
+ ContainerType containerType, ExecutionType executionType,
+ long allocationRequestId, Set<String> allocationTags) {
ContainerTokenIdentifierProto.Builder builder =
ContainerTokenIdentifierProto.newBuilder();
if (containerID != null) {
@@ -166,7 +220,9 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
builder.setContainerType(convertToProtoFormat(containerType));
builder.setExecutionType(convertToProtoFormat(executionType));
builder.setAllocationRequestId(allocationRequestId);
-
+ if (allocationTags != null) {
+ builder.addAllAllocationTags(allocationTags);
+ }
proto = builder.build();
}
@@ -308,6 +364,13 @@ public class ContainerTokenIdentifier extends TokenIdentifier {
return CommonNodeLabelsManager.NO_LABEL;
}
+ public Set<String> getAllcationTags() {
+ if (proto.getAllocationTagsList() != null) {
+ return new HashSet<>(proto.getAllocationTagsList());
+ }
+ return Collections.EMPTY_SET;
+ }
+
// TODO: Needed?
@InterfaceAudience.Private
public static class Renewer extends Token.TrivialRenewer {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
index d8288ac..9aabd48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/yarn_security_token.proto
@@ -54,6 +54,7 @@ message ContainerTokenIdentifierProto {
optional ExecutionTypeProto executionType = 13 [default = GUARANTEED];
optional int32 version = 14 [default = 0];
optional int64 allocation_request_id = 15 [default = -1];
+ repeated string allocation_tags = 16;
}
message ClientToAMTokenIdentifierProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
index 1a095f2..77b3df6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NMContainerStatus.java
@@ -27,6 +27,9 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.util.Records;
+import java.util.Collections;
+import java.util.Set;
+
/**
* NMContainerStatus includes the current information of a container. This
* record is used by YARN only, whereas {@link ContainerStatus} is used both
@@ -161,4 +164,15 @@ public abstract class NMContainerStatus {
}
public void setExecutionType(ExecutionType executionType) { }
+
+ /**
+ * Get and set the Allocation tags associated with the container.
+ */
+ public Set<String> getAllocationTags() {
+ return Collections.EMPTY_SET;
+ }
+
+ public void setAllocationTags(Set<String> allocationTags) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
index 8ed02fa..14f2241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NMContainerStatusPBImpl.java
@@ -37,6 +37,9 @@ import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerSta
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import java.util.HashSet;
+import java.util.Set;
+
public class NMContainerStatusPBImpl extends NMContainerStatus {
NMContainerStatusProto proto = NMContainerStatusProto
@@ -47,6 +50,7 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
private ContainerId containerId = null;
private Resource resource = null;
private Priority priority = null;
+ private Set<String> allocationTags = null;
public NMContainerStatusPBImpl() {
builder = NMContainerStatusProto.newBuilder();
@@ -91,8 +95,11 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
.append("Diagnostics: ").append(getDiagnostics()).append(", ")
.append("ExitStatus: ").append(getContainerExitStatus()).append(", ")
.append("NodeLabelExpression: ").append(getNodeLabelExpression())
+ .append(", ")
.append("Priority: ").append(getPriority()).append(", ")
.append("AllocationRequestId: ").append(getAllocationRequestId())
+ .append(", ")
+ .append("AllocationTags: ").append(getAllocationTags()).append(", ")
.append("]");
return sb.toString();
}
@@ -283,6 +290,28 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
builder.setAllocationRequestId(allocationRequestId);
}
+ private void initAllocationTags() {
+ if (this.allocationTags != null) {
+ return;
+ }
+ NMContainerStatusProtoOrBuilder p = viaProto ? proto : builder;
+ this.allocationTags = new HashSet<>();
+ this.allocationTags.addAll(p.getAllocationTagsList());
+ }
+
+ @Override
+ public Set<String> getAllocationTags() {
+ initAllocationTags();
+ return this.allocationTags;
+ }
+
+ @Override
+ public void setAllocationTags(Set<String> allocationTags) {
+ maybeInitBuilder();
+ builder.clearAllocationTags();
+ this.allocationTags = allocationTags;
+ }
+
private void mergeLocalToBuilder() {
if (this.containerId != null
&& !((ContainerIdPBImpl) containerId).getProto().equals(
@@ -297,6 +326,10 @@ public class NMContainerStatusPBImpl extends NMContainerStatus {
if (this.priority != null) {
builder.setPriority(convertToProtoFormat(this.priority));
}
+ if (this.allocationTags != null) {
+ builder.clearAllocationTags();
+ builder.addAllAllocationTags(this.allocationTags);
+ }
}
private void mergeLocalToProto() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 8c4fc69..e782cc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -177,6 +177,7 @@ message NMContainerStatusProto {
optional int32 version = 9;
optional ExecutionTypeProto executionType = 10 [default = GUARANTEED];
optional int64 allocation_request_id = 11 [default = -1];
+ repeated string allocation_tags = 12;
}
message SCMUploaderNotifyRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 44bfc68..6b4d517 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -451,7 +451,8 @@ public class ContainerManagerImpl extends CompositeService implements
originalToken.getLogAggregationContext(),
originalToken.getNodeLabelExpression(),
originalToken.getContainerType(), originalToken.getExecutionType(),
- originalToken.getAllocationRequestId());
+ originalToken.getAllocationRequestId(),
+ originalToken.getAllcationTags());
} else {
token = BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index f95c2a0..0220d6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -844,14 +844,17 @@ public class ContainerImpl implements Container {
public NMContainerStatus getNMContainerStatus() {
this.readLock.lock();
try {
- return NMContainerStatus.newInstance(this.containerId,
- this.version, getCurrentState(), getResource(),
- diagnostics.toString(), exitCode,
- containerTokenIdentifier.getPriority(),
- containerTokenIdentifier.getCreationTime(),
- containerTokenIdentifier.getNodeLabelExpression(),
- containerTokenIdentifier.getExecutionType(),
- containerTokenIdentifier.getAllocationRequestId());
+ NMContainerStatus status =
+ NMContainerStatus.newInstance(this.containerId,
+ this.version, getCurrentState(), getResource(),
+ diagnostics.toString(), exitCode,
+ containerTokenIdentifier.getPriority(),
+ containerTokenIdentifier.getCreationTime(),
+ containerTokenIdentifier.getNodeLabelExpression(),
+ containerTokenIdentifier.getExecutionType(),
+ containerTokenIdentifier.getAllocationRequestId());
+ status.setAllocationTags(containerTokenIdentifier.getAllcationTags());
+ return status;
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 2c4ef7b..563df0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -530,12 +530,18 @@ public class RMContainerImpl implements RMContainer {
RMContainerEvent event) {
NMContainerStatus report =
((RMContainerRecoverEvent) event).getContainerReport();
+ // Set the allocation tags from the
+ container.setAllocationTags(report.getAllocationTags());
+ // Notify AllocationTagsManager
+ container.rmContext.getAllocationTagsManager().addContainer(
+ container.getNodeId(), container.getContainerId(),
+ container.getAllocationTags());
+
if (report.getContainerState().equals(ContainerState.COMPLETE)) {
ContainerStatus status =
ContainerStatus.newInstance(report.getContainerId(),
report.getContainerState(), report.getDiagnostics(),
report.getContainerExitStatus());
-
new FinishedTransition().transition(container,
new RMContainerFinishedEvent(container.getContainerId(), status,
RMContainerEventType.FINISHED));
@@ -577,7 +583,7 @@ public class RMContainerImpl implements RMContainer {
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
- // Notify placementManager
+ // Notify AllocationTagsManager
container.rmContext.getAllocationTagsManager().addContainer(
container.getNodeId(), container.getContainerId(),
container.getAllocationTags());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f02f113..88a9049 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -670,7 +670,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
container.getPriority(), rmContainer.getCreationTime(),
this.logAggregationContext, rmContainer.getNodeLabelExpression(),
containerType, container.getExecutionType(),
- container.getAllocationRequestId()));
+ container.getAllocationRequestId(),
+ rmContainer.getAllocationTags()));
updateNMToken(container);
} catch (IllegalArgumentException e) {
// DNS might be down, skip returning this container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
index 191900b..945d89e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
+import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -166,25 +168,14 @@ public class RMContainerTokenSecretManager extends
}
}
- /**
- * Helper function for creating ContainerTokens.
- *
- * @param containerId Container Id
- * @param containerVersion Container Version
- * @param nodeId Node Id
- * @param appSubmitter App Submitter
- * @param capability Capability
- * @param priority Priority
- * @param createTime Create Time
- * @return the container-token
- */
+ @VisibleForTesting
public Token createContainerToken(ContainerId containerId,
int containerVersion, NodeId nodeId, String appSubmitter,
Resource capability, Priority priority, long createTime) {
return createContainerToken(containerId, containerVersion, nodeId,
appSubmitter, capability, priority, createTime,
null, null, ContainerType.TASK,
- ExecutionType.GUARANTEED, -1);
+ ExecutionType.GUARANTEED, -1, null);
}
/**
@@ -209,7 +200,7 @@ public class RMContainerTokenSecretManager extends
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExpression,
ContainerType containerType, ExecutionType execType,
- long allocationRequestId) {
+ long allocationRequestId, Set<String> allocationTags) {
byte[] password;
ContainerTokenIdentifier tokenIdentifier;
long expiryTimeStamp =
@@ -224,7 +215,7 @@ public class RMContainerTokenSecretManager extends
this.currentMasterKey.getMasterKey().getKeyId(),
ResourceManager.getClusterTimeStamp(), priority, createTime,
logAggregationContext, nodeLabelExpression, containerType,
- execType, allocationRequestId);
+ execType, allocationRequestId, allocationTags);
password = this.createPassword(tokenIdentifier);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/308fc39e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index 6f54d47..25e535a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -296,12 +297,12 @@ public class TestContainerAllocation {
Resource capability, Priority priority, long createTime,
LogAggregationContext logAggregationContext, String nodeLabelExp,
ContainerType containerType, ExecutionType executionType,
- long allocationRequestId) {
+ long allocationRequestId, Set<String> allocationTags) {
numRetries++;
return super.createContainerToken(containerId, containerVersion,
nodeId, appSubmitter, capability, priority, createTime,
logAggregationContext, nodeLabelExp, containerType,
- executionType, allocationRequestId);
+ executionType, allocationRequestId, allocationTags);
}
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/11] hadoop git commit: YARN-7612. Add Processor Framework for
Rich Placement Constraints. (asuresh)
Posted by as...@apache.org.
YARN-7612. Add Processor Framework for Rich Placement Constraints. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e802a6c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e802a6c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e802a6c0
Branch: refs/heads/YARN-6592
Commit: e802a6c06cb46462ad84b77e4df663d075c02316
Parents: f650d39
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 22 15:51:20 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:51:28 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 26 ++
.../src/main/resources/yarn-default.xml | 30 ++
.../ApplicationMasterService.java | 15 +
.../rmcontainer/RMContainerImpl.java | 7 +-
.../scheduler/capacity/CapacityScheduler.java | 2 +
.../constraint/processor/BatchedRequests.java | 105 +++++
.../processor/NodeCandidateSelector.java | 38 ++
.../processor/PlacementDispatcher.java | 145 +++++++
.../processor/PlacementProcessor.java | 343 ++++++++++++++++
.../processor/SamplePlacementAlgorithm.java | 144 +++++++
.../constraint/processor/package-info.java | 29 ++
.../yarn/server/resourcemanager/MockAM.java | 26 ++
.../yarn/server/resourcemanager/MockRM.java | 14 +
.../constraint/TestPlacementProcessor.java | 394 +++++++++++++++++++
14 files changed, 1316 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1b6bd0e..03c24d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -529,6 +529,32 @@ public class YarnConfiguration extends Configuration {
/** The class to use as the resource scheduler.*/
public static final String RM_SCHEDULER =
RM_PREFIX + "scheduler.class";
+
+ /** Placement Algorithm. */
+ public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
+ RM_PREFIX + "placement-constraints.algorithm.class";
+
+ public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED =
+ RM_PREFIX + "placement-constraints.enabled";
+
+ public static final boolean DEFAULT_RM_PLACEMENT_CONSTRAINTS_ENABLED = true;
+
+ public static final String RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS =
+ RM_PREFIX + "placement-constraints.retry-attempts";
+
+ public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS = 3;
+
+ public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE =
+ RM_PREFIX + "placement-constraints.algorithm.pool-size";
+
+ public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE =
+ 1;
+
+ public static final String RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE =
+ RM_PREFIX + "placement-constraints.scheduler.pool-size";
+
+ public static final int DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE =
+ 1;
public static final String DEFAULT_RM_SCHEDULER =
"org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d450eca..0285069 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -131,6 +131,36 @@
</property>
<property>
+ <description>Enable Constraint Placement.</description>
+ <name>yarn.resourcemanager.placement-constraints.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>Number of times to retry placing of rejected SchedulingRequests</description>
+ <name>yarn.resourcemanager.placement-constraints.retry-attempts</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <description>Constraint Placement Algorithm to be used.</description>
+ <name>yarn.resourcemanager.placement-constraints.algorithm.class</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.SamplePlacementAlgorithm</value>
+ </property>
+
+ <property>
+ <description>Threadpool size for the Algorithm used for placement constraint processing.</description>
+ <name>yarn.resourcemanager.placement-constraints.algorithm.pool-size</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <description>Threadpool size for the Scheduler invocation phase of placement constraint processing.</description>
+ <name>yarn.resourcemanager.placement-constraints.scheduler.pool-size</name>
+ <value>1</value>
+ </property>
+
+ <property>
<description>
Comma separated class names of ApplicationMasterServiceProcessor
implementations. The processors will be applied in the order
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 90c42be..aa1177d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.PlacementProcessor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
@@ -114,11 +115,25 @@ public class ApplicationMasterService extends AbstractService implements
YarnConfiguration.RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+ initializeProcessingChain(conf);
+ }
+
+ private void initializeProcessingChain(Configuration conf) {
amsProcessingChain.init(rmContext, null);
+ boolean enablePlacementConstraints = conf.getBoolean(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ENABLED,
+ YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_ENABLED);
+ if (enablePlacementConstraints) {
+ amsProcessingChain.addProcessor(new PlacementProcessor());
+ }
List<ApplicationMasterServiceProcessor> processors = getProcessorList(conf);
if (processors != null) {
Collections.reverse(processors);
for (ApplicationMasterServiceProcessor p : processors) {
+ // Ensure only single instance of PlacementProcessor is included
+ if (enablePlacementConstraints && p instanceof PlacementProcessor) {
+ continue;
+ }
this.amsProcessingChain.addProcessor(p);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 184cdfc..c873509 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -190,8 +190,7 @@ public class RMContainerImpl implements RMContainer {
private boolean isExternallyAllocated;
private SchedulerRequestKey allocatedSchedulerKey;
- // TODO, set it when container allocated by scheduler (From SchedulingRequest)
- private Set<String> allocationTags = null;
+ private volatile Set<String> allocationTags = null;
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
@@ -510,6 +509,10 @@ public class RMContainerImpl implements RMContainer {
return allocationTags;
}
+ public void setAllocationTags(Set<String> tags) {
+ this.allocationTags = tags;
+ }
+
private static class BaseTransition implements
SingleArcTransition<RMContainerImpl, RMContainerEvent> {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index d92ce58..f03d7d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2539,6 +2539,8 @@ public class CapacityScheduler extends
SchedulerRequestKey.extractFrom(container),
appAttempt.getApplicationAttemptId(), container.getNodeId(),
appAttempt.getUser(), rmContext, false);
+ ((RMContainerImpl)rmContainer).setAllocationTags(
+ new HashSet<>(schedulingRequest.getAllocationTags()));
allocated = new ContainerAllocationProposal<>(
getSchedulerContainer(rmContainer, true),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
new file mode 100644
index 0000000..fe92d2f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A grouping of Scheduling Requests which are sent to the PlacementAlgorithm
+ * to place as a batch. The placement algorithm tends to give more optimal
+ * placements if more requests are batched together.
+ */
+class BatchedRequests implements ConstraintPlacementAlgorithmInput {
+
+ // PlacementAlgorithmOutput attempt - the number of times the requests in this
+ // batch has been placed but was rejected by the scheduler.
+ private final int placementAttempt;
+
+ private final ApplicationId applicationId;
+ private final Collection<SchedulingRequest> requests;
+ private final Map<String, Set<NodeId>> blacklist = new HashMap<>();
+
+ BatchedRequests(ApplicationId applicationId,
+ Collection<SchedulingRequest> requests, int attempt) {
+ this.applicationId = applicationId;
+ this.requests = requests;
+ this.placementAttempt = attempt;
+ }
+
+ /**
+ * Get Application Id.
+ * @return Application Id.
+ */
+ ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ /**
+ * Get Collection of SchedulingRequests in this batch.
+ * @return Collection of Scheduling Requests.
+ */
+ @Override
+ public Collection<SchedulingRequest> getSchedulingRequests() {
+ return requests;
+ }
+
+ /**
+ * Add a Scheduling request to the batch.
+ * @param req Scheduling Request.
+ */
+ void addToBatch(SchedulingRequest req) {
+ requests.add(req);
+ }
+
+ void addToBlacklist(Set<String> tags, SchedulerNode node) {
+ if (tags != null && !tags.isEmpty()) {
+ // We are currently assuming a single allocation tag
+ // per scheduler request currently.
+ blacklist.computeIfAbsent(tags.iterator().next(),
+ k -> new HashSet<>()).add(node.getNodeID());
+ }
+ }
+
+ /**
+ * Get placement attempt.
+ * @return PlacementAlgorithmOutput placement Attempt.
+ */
+ int getPlacementAttempt() {
+ return placementAttempt;
+ }
+
+ /**
+ * Get any blacklisted nodes associated with tag.
+ * @param tag Tag.
+ * @return Set of blacklisted Nodes.
+ */
+ Set<NodeId> getBlacklist(String tag) {
+ return blacklist.getOrDefault(tag, Collections.EMPTY_SET);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/NodeCandidateSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/NodeCandidateSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/NodeCandidateSelector.java
new file mode 100644
index 0000000..4299050
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/NodeCandidateSelector.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeFilter;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+
+import java.util.List;
+
+/**
+ * A read only implementation of the ClusterNodeTracker which exposes a method
+ * to simply return a filtered list of nodes.
+ */
+public interface NodeCandidateSelector {
+
+ /**
+ * Select a list of nodes given a filter.
+ * @param filter a NodeFilter.
+ * @return List of SchedulerNodes.
+ */
+ List<SchedulerNode> selectNodes(NodeFilter filter);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementDispatcher.java
new file mode 100644
index 0000000..6a00ba8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementDispatcher.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutput;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutputCollector;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * This class initializes the Constraint Placement Algorithm. It dispatches
+ * input to the algorithm and collects output from it.
+ */
+class PlacementDispatcher implements
+ ConstraintPlacementAlgorithmOutputCollector {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PlacementDispatcher.class);
+ private ConstraintPlacementAlgorithm algorithm;
+ private ExecutorService algorithmThreadPool;
+
+ private Map<ApplicationId, List<PlacedSchedulingRequest>>
+ placedRequests = new ConcurrentHashMap<>();
+ private Map<ApplicationId, List<SchedulingRequest>>
+ rejectedRequests = new ConcurrentHashMap<>();
+
+ public void init(RMContext rmContext,
+ ConstraintPlacementAlgorithm placementAlgorithm, int poolSize) {
+ LOG.info("Initializing Constraint Placement Planner:");
+ this.algorithm = placementAlgorithm;
+ this.algorithm.init(rmContext);
+ this.algorithmThreadPool = Executors.newFixedThreadPool(poolSize);
+ }
+
+ void dispatch(final BatchedRequests batchedRequests) {
+ final ConstraintPlacementAlgorithmOutputCollector collector = this;
+ Runnable placingTask = () -> {
+ LOG.debug("Got [{}] requests to place from application [{}].. " +
+ "Attempt count [{}]",
+ batchedRequests.getSchedulingRequests().size(),
+ batchedRequests.getApplicationId(),
+ batchedRequests.getPlacementAttempt());
+ algorithm.place(batchedRequests, collector);
+ };
+ this.algorithmThreadPool.submit(placingTask);
+ }
+
+ public List<PlacedSchedulingRequest> pullPlacedRequests(
+ ApplicationId applicationId) {
+ List<PlacedSchedulingRequest> placedReqs =
+ this.placedRequests.get(applicationId);
+ if (placedReqs != null && !placedReqs.isEmpty()) {
+ List<PlacedSchedulingRequest> retList = new ArrayList<>();
+ synchronized (placedReqs) {
+ if (placedReqs.size() > 0) {
+ retList.addAll(placedReqs);
+ placedReqs.clear();
+ }
+ }
+ return retList;
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+ public List<SchedulingRequest> pullRejectedRequests(
+ ApplicationId applicationId) {
+ List<SchedulingRequest> rejectedReqs =
+ this.rejectedRequests.get(applicationId);
+ if (rejectedReqs != null && !rejectedReqs.isEmpty()) {
+ List<SchedulingRequest> retList = new ArrayList<>();
+ synchronized (rejectedReqs) {
+ if (rejectedReqs.size() > 0) {
+ retList.addAll(rejectedReqs);
+ rejectedReqs.clear();
+ }
+ }
+ return retList;
+ }
+ return Collections.EMPTY_LIST;
+ }
+
+ void clearApplicationState(ApplicationId applicationId) {
+ placedRequests.remove(applicationId);
+ rejectedRequests.remove(applicationId);
+ }
+
+ @Override
+ public void collect(ConstraintPlacementAlgorithmOutput placement) {
+ if (!placement.getPlacedRequests().isEmpty()) {
+ List<PlacedSchedulingRequest> processed =
+ placedRequests.computeIfAbsent(
+ placement.getApplicationId(), k -> new ArrayList<>());
+ synchronized (processed) {
+ LOG.debug(
+ "Planning Algorithm has placed for application [{}]" +
+ " the following [{}]", placement.getApplicationId(),
+ placement.getPlacedRequests());
+ for (PlacedSchedulingRequest esr :
+ placement.getPlacedRequests()) {
+ processed.add(esr);
+ }
+ }
+ }
+ if (!placement.getRejectedRequests().isEmpty()) {
+ List<SchedulingRequest> rejected =
+ rejectedRequests.computeIfAbsent(
+ placement.getApplicationId(), k -> new ArrayList());
+ LOG.warn(
+ "Planning Algorithm has rejected for application [{}]" +
+ " the following [{}]", placement.getApplicationId(),
+ placement.getRejectedRequests());
+ synchronized (rejected) {
+ rejected.addAll(placement.getRejectedRequests());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
new file mode 100644
index 0000000..d613d4e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
@@ -0,0 +1,343 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
+
+import org.apache.hadoop.yarn.ams.ApplicationMasterServiceContext;
+import org.apache.hadoop.yarn.ams.ApplicationMasterServiceProcessor;
+import org.apache.hadoop.yarn.ams.ApplicationMasterServiceUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
+import org.apache.hadoop.yarn.api.records.RejectionReason;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.SchedulingResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.Collectors;
+
+/**
+ * An ApplicationMasterService Processor that performs Constrained placement of
+ * Scheduling Requests. It does the following:
+ * 1. All initialization.
+ * 2. Intercepts placement constraints from the register call and adds it to
+ * the placement constraint manager.
+ * 3. Dispatches Scheduling Requests to the Planner.
+ */
+public class PlacementProcessor implements ApplicationMasterServiceProcessor {
+
+ /**
+ * Wrapper over the SchedulingResponse that wires in the placement attempt
+ * and last attempted Node.
+ */
+ static final class Response extends SchedulingResponse {
+
+ private final int placementAttempt;
+ private final SchedulerNode attemptedNode;
+
+ private Response(boolean isSuccess, ApplicationId applicationId,
+ SchedulingRequest schedulingRequest, int placementAttempt,
+ SchedulerNode attemptedNode) {
+ super(isSuccess, applicationId, schedulingRequest);
+ this.placementAttempt = placementAttempt;
+ this.attemptedNode = attemptedNode;
+ }
+ }
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PlacementProcessor.class);
+ private PlacementConstraintManager constraintManager;
+ private ApplicationMasterServiceProcessor nextAMSProcessor;
+
+ private AbstractYarnScheduler scheduler;
+ private ExecutorService schedulingThreadPool;
+ private int retryAttempts;
+ private Map<ApplicationId, List<BatchedRequests>> requestsToRetry =
+ new ConcurrentHashMap<>();
+ private Map<ApplicationId, List<SchedulingRequest>> requestsToReject =
+ new ConcurrentHashMap<>();
+
+ private PlacementDispatcher placementDispatcher;
+
+
+ @Override
+ public void init(ApplicationMasterServiceContext amsContext,
+ ApplicationMasterServiceProcessor nextProcessor) {
+ LOG.info("Initializing Constraint Placement Processor:");
+ this.nextAMSProcessor = nextProcessor;
+ this.constraintManager =
+ ((RMContextImpl)amsContext).getPlacementConstraintManager();
+
+ this.scheduler =
+ (AbstractYarnScheduler)((RMContextImpl)amsContext).getScheduler();
+ // Only the first class is considered - even if a comma separated
+ // list is provided. (This is for simplicity, since getInstances does a
+ // lot of good things by handling things correctly)
+ List<ConstraintPlacementAlgorithm> instances =
+ ((RMContextImpl) amsContext).getYarnConfiguration().getInstances(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS,
+ ConstraintPlacementAlgorithm.class);
+ ConstraintPlacementAlgorithm algorithm = null;
+ if (instances != null && !instances.isEmpty()) {
+ algorithm = instances.get(0);
+ } else {
+ algorithm = new SamplePlacementAlgorithm();
+ }
+ LOG.info("Planning Algorithm [{}]", algorithm.getClass().getName());
+
+ int algoPSize = ((RMContextImpl) amsContext).getYarnConfiguration().getInt(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE,
+ YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE);
+ this.placementDispatcher = new PlacementDispatcher();
+ this.placementDispatcher.init(
+ ((RMContextImpl)amsContext), algorithm, algoPSize);
+ LOG.info("Planning Algorithm pool size [{}]", algoPSize);
+
+ int schedPSize = ((RMContextImpl) amsContext).getYarnConfiguration().getInt(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE,
+ YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_SCHEDULER_POOL_SIZE);
+ this.schedulingThreadPool = Executors.newFixedThreadPool(schedPSize);
+ LOG.info("Scheduler pool size [{}]", schedPSize);
+
+ // Number of times a request that is not satisfied by the scheduler
+ // can be retried.
+ this.retryAttempts =
+ ((RMContextImpl) amsContext).getYarnConfiguration().getInt(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS);
+ LOG.info("Num retry attempts [{}]", this.retryAttempts);
+ }
+
+ @Override
+ public void registerApplicationMaster(ApplicationAttemptId appAttemptId,
+ RegisterApplicationMasterRequest request,
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
+ Map<Set<String>, PlacementConstraint> appPlacementConstraints =
+ request.getPlacementConstraints();
+ processPlacementConstraints(
+ appAttemptId.getApplicationId(), appPlacementConstraints);
+ nextAMSProcessor.registerApplicationMaster(appAttemptId, request, response);
+ }
+
+ private void processPlacementConstraints(ApplicationId applicationId,
+ Map<Set<String>, PlacementConstraint> appPlacementConstraints) {
+ if (appPlacementConstraints != null && !appPlacementConstraints.isEmpty()) {
+ LOG.info("Constraints added for application [{}] against tags [{}]",
+ applicationId, appPlacementConstraints);
+ constraintManager.registerApplication(
+ applicationId, appPlacementConstraints);
+ }
+ }
+
+ @Override
+ public void allocate(ApplicationAttemptId appAttemptId,
+ AllocateRequest request, AllocateResponse response) throws YarnException {
+ List<SchedulingRequest> schedulingRequests =
+ request.getSchedulingRequests();
+ dispatchRequestsForPlacement(appAttemptId, schedulingRequests);
+ reDispatchRetryableRequests(appAttemptId);
+ schedulePlacedRequests(appAttemptId);
+
+ nextAMSProcessor.allocate(appAttemptId, request, response);
+
+ handleRejectedRequests(appAttemptId, response);
+ }
+
+ private void dispatchRequestsForPlacement(ApplicationAttemptId appAttemptId,
+ List<SchedulingRequest> schedulingRequests) {
+ if (schedulingRequests != null && !schedulingRequests.isEmpty()) {
+ this.placementDispatcher.dispatch(
+ new BatchedRequests(appAttemptId.getApplicationId(),
+ schedulingRequests, 1));
+ }
+ }
+
+ private void reDispatchRetryableRequests(ApplicationAttemptId appAttId) {
+ List<BatchedRequests> reqsToRetry =
+ this.requestsToRetry.get(appAttId.getApplicationId());
+ if (reqsToRetry != null && !reqsToRetry.isEmpty()) {
+ synchronized (reqsToRetry) {
+ for (BatchedRequests bReq: reqsToRetry) {
+ this.placementDispatcher.dispatch(bReq);
+ }
+ reqsToRetry.clear();
+ }
+ }
+ }
+
+ private void schedulePlacedRequests(ApplicationAttemptId appAttemptId) {
+ ApplicationId applicationId = appAttemptId.getApplicationId();
+ List<PlacedSchedulingRequest> placedSchedulingRequests =
+ this.placementDispatcher.pullPlacedRequests(applicationId);
+ for (PlacedSchedulingRequest placedReq : placedSchedulingRequests) {
+ SchedulingRequest sReq = placedReq.getSchedulingRequest();
+ for (SchedulerNode node : placedReq.getNodes()) {
+ final SchedulingRequest sReqClone =
+ SchedulingRequest.newInstance(sReq.getAllocationRequestId(),
+ sReq.getPriority(), sReq.getExecutionType(),
+ sReq.getAllocationTags(),
+ ResourceSizing.newInstance(
+ sReq.getResourceSizing().getResources()),
+ sReq.getPlacementConstraint());
+ SchedulerApplicationAttempt applicationAttempt =
+ this.scheduler.getApplicationAttempt(appAttemptId);
+ Runnable task = () -> {
+ boolean success =
+ scheduler.attemptAllocationOnNode(
+ applicationAttempt, sReqClone, node);
+ if (!success) {
+ LOG.warn("Unsuccessful allocation attempt [{}] for [{}]",
+ placedReq.getPlacementAttempt(), sReqClone);
+ }
+ handleSchedulingResponse(
+ new Response(success, applicationId, sReqClone,
+ placedReq.getPlacementAttempt(), node));
+ };
+ this.schedulingThreadPool.submit(task);
+ }
+ }
+ }
+
+ private void handleRejectedRequests(ApplicationAttemptId appAttemptId,
+ AllocateResponse response) {
+ List<SchedulingRequest> rejectedRequests =
+ this.placementDispatcher.pullRejectedRequests(
+ appAttemptId.getApplicationId());
+ if (rejectedRequests != null && !rejectedRequests.isEmpty()) {
+ LOG.warn("Following requests of [{}] were rejected by" +
+ " the PlacementAlgorithmOutput Algorithm: {}",
+ appAttemptId.getApplicationId(), rejectedRequests);
+ ApplicationMasterServiceUtils.addToRejectedSchedulingRequests(response,
+ rejectedRequests.stream()
+ .map(sr -> RejectedSchedulingRequest.newInstance(
+ RejectionReason.COULD_NOT_PLACE_ON_NODE, sr))
+ .collect(Collectors.toList()));
+ }
+ rejectedRequests =
+ this.requestsToReject.get(appAttemptId.getApplicationId());
+ if (rejectedRequests != null && !rejectedRequests.isEmpty()) {
+ synchronized (rejectedRequests) {
+ LOG.warn("Following requests of [{}] exhausted all retry attempts " +
+ "trying to schedule on placed node: {}",
+ appAttemptId.getApplicationId(), rejectedRequests);
+ ApplicationMasterServiceUtils.addToRejectedSchedulingRequests(response,
+ rejectedRequests.stream()
+ .map(sr -> RejectedSchedulingRequest.newInstance(
+ RejectionReason.COULD_NOT_SCHEDULE_ON_NODE, sr))
+ .collect(Collectors.toList()));
+ rejectedRequests.clear();
+ }
+ }
+ }
+
+ @Override
+ public void finishApplicationMaster(ApplicationAttemptId appAttemptId,
+ FinishApplicationMasterRequest request,
+ FinishApplicationMasterResponse response) {
+ constraintManager.unregisterApplication(appAttemptId.getApplicationId());
+ placementDispatcher.clearApplicationState(appAttemptId.getApplicationId());
+ requestsToReject.remove(appAttemptId.getApplicationId());
+ requestsToRetry.remove(appAttemptId.getApplicationId());
+ nextAMSProcessor.finishApplicationMaster(appAttemptId, request, response);
+ }
+
+ private void handleSchedulingResponse(SchedulingResponse schedulerResponse) {
+ int placementAttempt = ((Response)schedulerResponse).placementAttempt;
+ // Retry this placement as it is not successful and we are still
+ // under max retry. The req is batched with other unsuccessful
+ // requests from the same app
+ if (!schedulerResponse.isSuccess() && placementAttempt < retryAttempts) {
+ List<BatchedRequests> reqsToRetry =
+ requestsToRetry.computeIfAbsent(
+ schedulerResponse.getApplicationId(),
+ k -> new ArrayList<>());
+ synchronized (reqsToRetry) {
+ addToRetryList(schedulerResponse, placementAttempt, reqsToRetry);
+ }
+ LOG.warn("Going to retry request for application [{}] after [{}]" +
+ " attempts: [{}]", schedulerResponse.getApplicationId(),
+ placementAttempt, schedulerResponse.getSchedulingRequest());
+ } else {
+ if (!schedulerResponse.isSuccess()) {
+ LOG.warn("Not retrying request for application [{}] after [{}]" +
+ " attempts: [{}]", schedulerResponse.getApplicationId(),
+ placementAttempt, schedulerResponse.getSchedulingRequest());
+ List<SchedulingRequest> reqsToReject =
+ requestsToReject.computeIfAbsent(
+ schedulerResponse.getApplicationId(),
+ k -> new ArrayList<>());
+ synchronized (reqsToReject) {
+ reqsToReject.add(schedulerResponse.getSchedulingRequest());
+ }
+ }
+ }
+ }
+
+ private void addToRetryList(SchedulingResponse schedulerResponse,
+ int placementAttempt, List<BatchedRequests> reqsToRetry) {
+ boolean isAdded = false;
+ for (BatchedRequests br : reqsToRetry) {
+ if (br.getPlacementAttempt() == placementAttempt + 1) {
+ br.addToBatch(schedulerResponse.getSchedulingRequest());
+ br.addToBlacklist(
+ schedulerResponse.getSchedulingRequest().getAllocationTags(),
+ ((Response) schedulerResponse).attemptedNode);
+ isAdded = true;
+ break;
+ }
+ }
+ if (!isAdded) {
+ BatchedRequests br =
+ new BatchedRequests(schedulerResponse.getApplicationId(),
+ Collections.singleton(
+ schedulerResponse.getSchedulingRequest()),
+ placementAttempt + 1);
+ reqsToRetry.add(br);
+ br.addToBlacklist(
+ schedulerResponse.getSchedulingRequest().getAllocationTags(),
+ ((Response) schedulerResponse).attemptedNode);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
new file mode 100644
index 0000000..8d49801
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SpecializedConstraintTransformer;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutput;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutputCollector;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Sample Test algorithm. Assumes anti-affinity always
+ * It also assumes the numAllocations in resource sizing is always = 1
+ *
+ * NOTE: This is just a sample implementation. Not be actually used
+ */
+public class SamplePlacementAlgorithm implements ConstraintPlacementAlgorithm {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SamplePlacementAlgorithm.class);
+
+ private AllocationTagsManager tagsManager;
+ private PlacementConstraintManager constraintManager;
+ private NodeCandidateSelector nodeSelector;
+
+ @Override
+ public void init(RMContext rmContext) {
+ this.tagsManager = rmContext.getAllocationTagsManager();
+ this.constraintManager = rmContext.getPlacementConstraintManager();
+ this.nodeSelector =
+ filter -> ((AbstractYarnScheduler)(rmContext)
+ .getScheduler()).getNodes(filter);
+ }
+
+ @Override
+ public void place(ConstraintPlacementAlgorithmInput input,
+ ConstraintPlacementAlgorithmOutputCollector collector) {
+ BatchedRequests requests = (BatchedRequests)input;
+ ConstraintPlacementAlgorithmOutput resp =
+ new ConstraintPlacementAlgorithmOutput(requests.getApplicationId());
+ List<SchedulerNode> allNodes = nodeSelector.selectNodes(null);
+ Map<String, List<SchedulingRequest>> tagIndexedRequests = new HashMap<>();
+ requests.getSchedulingRequests()
+ .stream()
+ .filter(r -> r.getAllocationTags() != null)
+ .forEach(
+ req -> req.getAllocationTags().forEach(
+ tag -> tagIndexedRequests.computeIfAbsent(tag,
+ k -> new ArrayList<>()).add(req))
+ );
+ for (Map.Entry<String, List<SchedulingRequest>> entry :
+ tagIndexedRequests.entrySet()) {
+ String tag = entry.getKey();
+ PlacementConstraint constraint =
+ constraintManager.getConstraint(requests.getApplicationId(),
+ Collections.singleton(tag));
+ if (constraint != null) {
+ // Currently works only for simple anti-affinity
+ // NODE scope target expressions
+ SpecializedConstraintTransformer transformer =
+ new SpecializedConstraintTransformer(constraint);
+ PlacementConstraint transform = transformer.transform();
+ TargetConstraint targetConstraint =
+ (TargetConstraint) transform.getConstraintExpr();
+ // Assume a single target expression tag;
+ // The Sample Algorithm assumes a constraint will always be a simple
+ // Target Constraint with a single entry in the target set.
+ // As mentioned in the class javadoc - This algorithm should be
+ // used mostly for testing and validating end-2-end workflow.
+ String targetTag =
+ targetConstraint.getTargetExpressions().iterator().next()
+ .getTargetValues().iterator().next();
+ // iterate over all nodes
+ Iterator<SchedulerNode> nodeIter = allNodes.iterator();
+ List<SchedulingRequest> schedulingRequests = entry.getValue();
+ Iterator<SchedulingRequest> reqIter = schedulingRequests.iterator();
+ while (reqIter.hasNext()) {
+ SchedulingRequest sReq = reqIter.next();
+ int numAllocs = sReq.getResourceSizing().getNumAllocations();
+ while (numAllocs > 0 && nodeIter.hasNext()) {
+ SchedulerNode node = nodeIter.next();
+ long nodeCardinality = 0;
+ try {
+ nodeCardinality = tagsManager.getNodeCardinality(
+ node.getNodeID(), requests.getApplicationId(),
+ targetTag);
+ if (nodeCardinality == 0 &&
+ !requests.getBlacklist(tag).contains(node.getNodeID())) {
+ numAllocs--;
+ sReq.getResourceSizing().setNumAllocations(numAllocs);
+ PlacedSchedulingRequest placedReq =
+ new PlacedSchedulingRequest(sReq);
+ placedReq.setPlacementAttempt(requests.getPlacementAttempt());
+ placedReq.getNodes().add(node);
+ resp.getPlacedRequests().add(placedReq);
+ }
+ } catch (InvalidAllocationTagsQueryException e) {
+ LOG.warn("Got exception from TagManager !", e);
+ }
+ }
+ }
+ }
+ }
+ // Add all requests whose numAllocations still > 0 to rejected list.
+ requests.getSchedulingRequests().stream()
+ .filter(sReq -> sReq.getResourceSizing().getNumAllocations() > 0)
+ .forEach(rejReq -> resp.getRejectedRequests().add(rejReq));
+ collector.collect(resp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/package-info.java
new file mode 100644
index 0000000..7090154
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package o.a.h.yarn.server.resourcemanager.scheduler.constraint.processor
+ * contains classes related to scheduling containers using placement
+ * processor.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index 12dfe18..975abe6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -21,7 +21,10 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.lang.reflect.UndeclaredThrowableException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -39,7 +42,9 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -57,6 +62,9 @@ public class MockAM {
private ApplicationMasterProtocol amRMProtocol;
private UserGroupInformation ugi;
private volatile AllocateResponse lastResponse;
+ private Map<Set<String>, PlacementConstraint> placementConstraints =
+ new HashMap<>();
+ private List<SchedulingRequest> schedulingRequests = new ArrayList<>();
private final List<ResourceRequest> requests = new ArrayList<ResourceRequest>();
private final List<ContainerId> releases = new ArrayList<ContainerId>();
@@ -93,6 +101,16 @@ public class MockAM {
return registerAppAttempt(true);
}
+ public void addPlacementConstraint(Set<String> tags,
+ PlacementConstraint constraint) {
+ placementConstraints.put(tags, constraint);
+ }
+
+ public MockAM addSchedulingRequest(List<SchedulingRequest> reqs) {
+ schedulingRequests.addAll(reqs);
+ return this;
+ }
+
public RegisterApplicationMasterResponse registerAppAttempt(boolean wait)
throws Exception {
if (wait) {
@@ -104,6 +122,9 @@ public class MockAM {
req.setHost("");
req.setRpcPort(1);
req.setTrackingUrl("");
+ if (!placementConstraints.isEmpty()) {
+ req.setPlacementConstraints(this.placementConstraints);
+ }
if (ugi == null) {
ugi = UserGroupInformation.createRemoteUser(
attemptId.toString());
@@ -247,12 +268,17 @@ public class MockAM {
}
+
public AllocateResponse allocate(
List<ResourceRequest> resourceRequest, List<ContainerId> releases)
throws Exception {
final AllocateRequest req =
AllocateRequest.newInstance(0, 0F, resourceRequest,
releases, null);
+ if (!schedulingRequests.isEmpty()) {
+ req.setSchedulingRequests(schedulingRequests);
+ schedulingRequests.clear();
+ }
return allocate(req);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 302f5b3..f0e4213 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -27,6 +27,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
@@ -65,6 +66,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -1240,6 +1242,18 @@ public class MockRM extends ResourceManager {
return am;
}
+ public static MockAM launchAndRegisterAM(RMApp app, MockRM rm, MockNM nm,
+ Map<Set<String>, PlacementConstraint> constraints) throws Exception {
+ MockAM am = launchAM(app, rm, nm);
+ for (Map.Entry<Set<String>, PlacementConstraint> e :
+ constraints.entrySet()) {
+ am.addPlacementConstraint(e.getKey(), e.getValue());
+ }
+ am.registerAppAttempt();
+ rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
+ return am;
+ }
+
public ApplicationReport getApplicationReport(ApplicationId appId)
throws YarnException, IOException {
ApplicationClientProtocol client = getClientRMService();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e802a6c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
new file mode 100644
index 0000000..db8ae15
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
@@ -0,0 +1,394 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
+import org.apache.hadoop.yarn.api.records.RejectionReason;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.lang.Thread.sleep;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+/**
+ * This tests end2end workflow of the constraint placement framework.
+ */
+public class TestPlacementProcessor {
+
+ private static final int GB = 1024;
+
+ private static final Log LOG =
+ LogFactory.getLog(TestPlacementProcessor.class);
+ private MockRM rm;
+ private DrainDispatcher dispatcher;
+
+ @Before
+ public void createAndStartRM() {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS, 1);
+ startRM(conf);
+ }
+
+ private void startRM(final YarnConfiguration conf) {
+ dispatcher = new DrainDispatcher();
+ rm = new MockRM(conf) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+ };
+ rm.start();
+ }
+
+ @After
+ public void stopRM() {
+ if (rm != null) {
+ rm.stop();
+ }
+ }
+
+ @Test(timeout = 300000)
+ public void testPlacement() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
+ Collections.singletonMap(
+ Collections.singleton("foo"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetNotIn(NODE, allocationTag("foo")))
+ ));
+ am1.addSchedulingRequest(
+ Arrays.asList(
+ schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 5, 1, 512, "foo"))
+ );
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+
+ // kick the scheduler
+
+ while (allocatedContainers.size() < 4) {
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ LOG.info("Waiting for containers to be created for app 1...");
+ sleep(1000);
+ allocResponse = am1.schedule();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ }
+
+ Assert.assertEquals(4, allocatedContainers.size());
+ Set<NodeId> nodeIds = allocatedContainers.stream()
+ .map(x -> x.getNodeId()).collect(Collectors.toSet());
+ // Ensure unique nodes
+ Assert.assertEquals(4, nodeIds.size());
+ }
+
+ @Test(timeout = 300000)
+ public void testSchedulerRejection() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
+ Collections.singletonMap(
+ Collections.singleton("foo"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetNotIn(NODE, allocationTag("foo")))
+ ));
+ am1.addSchedulingRequest(
+ Arrays.asList(
+ schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ // Ask for a container larger than the node
+ schedulingRequest(1, 4, 1, 5120, "foo"))
+ );
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ List<RejectedSchedulingRequest> rejectedReqs = new ArrayList<>();
+ int allocCount = 1;
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
+
+ // kick the scheduler
+
+ while (allocCount < 11) {
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ LOG.info("Waiting for containers to be created for app 1...");
+ sleep(1000);
+ allocResponse = am1.schedule();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
+ allocCount++;
+ if (rejectedReqs.size() > 0 && allocatedContainers.size() > 2) {
+ break;
+ }
+ }
+
+ Assert.assertEquals(3, allocatedContainers.size());
+ Set<NodeId> nodeIds = allocatedContainers.stream()
+ .map(x -> x.getNodeId()).collect(Collectors.toSet());
+ // Ensure unique nodes
+ Assert.assertEquals(3, nodeIds.size());
+ RejectedSchedulingRequest rej = rejectedReqs.get(0);
+ Assert.assertEquals(4, rej.getRequest().getAllocationRequestId());
+ Assert.assertEquals(RejectionReason.COULD_NOT_SCHEDULE_ON_NODE,
+ rej.getReason());
+ }
+
+ @Test(timeout = 300000)
+ public void testRePlacementAfterSchedulerRejection() throws Exception {
+ stopRM();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_RETRY_ATTEMPTS, 2);
+ startRM(conf);
+
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ MockNM nm5 = new MockNM("h5:1234", 8192, rm.getResourceTrackerService());
+ nodes.put(nm5.getNodeId(), nm5);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+ // No not register nm5 yet..
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
+ Collections.singletonMap(
+ Collections.singleton("foo"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetNotIn(NODE, allocationTag("foo")))
+ ));
+ am1.addSchedulingRequest(
+ Arrays.asList(
+ schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ // Ask for a container larger than the node
+ schedulingRequest(1, 4, 1, 5120, "foo"))
+ );
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ List<RejectedSchedulingRequest> rejectedReqs = new ArrayList<>();
+ int allocCount = 1;
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
+
+ // Register node5 only after first allocate - so the initial placement
+ // for the large schedReq goes to some other node..
+ nm5.registerNode();
+
+ // kick the scheduler
+ while (allocCount < 11) {
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ nm5.nodeHeartbeat(true);
+ LOG.info("Waiting for containers to be created for app 1...");
+ sleep(1000);
+ allocResponse = am1.schedule();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
+ allocCount++;
+ if (allocatedContainers.size() > 3) {
+ break;
+ }
+ }
+
+ Assert.assertEquals(4, allocatedContainers.size());
+ Set<NodeId> nodeIds = allocatedContainers.stream()
+ .map(x -> x.getNodeId()).collect(Collectors.toSet());
+ // Ensure unique nodes
+ Assert.assertEquals(4, nodeIds.size());
+ }
+
+ @Test(timeout = 300000)
+ public void testPlacementRejection() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
+ Collections.singletonMap(
+ Collections.singleton("foo"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetNotIn(NODE, allocationTag("foo")))
+ ));
+ am1.addSchedulingRequest(
+ Arrays.asList(
+ schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 4, 1, 512, "foo"),
+ // Ask for more containers than nodes
+ schedulingRequest(1, 5, 1, 512, "foo"))
+ );
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ List<RejectedSchedulingRequest> rejectedReqs = new ArrayList<>();
+ int allocCount = 1;
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
+
+ // kick the scheduler
+
+ while (allocCount < 11) {
+ nm1.nodeHeartbeat(true);
+ nm2.nodeHeartbeat(true);
+ nm3.nodeHeartbeat(true);
+ nm4.nodeHeartbeat(true);
+ LOG.info("Waiting for containers to be created for app 1...");
+ sleep(1000);
+ allocResponse = am1.schedule();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
+ allocCount++;
+ if (rejectedReqs.size() > 0 && allocatedContainers.size() > 3) {
+ break;
+ }
+ }
+
+ Assert.assertEquals(4, allocatedContainers.size());
+ Set<NodeId> nodeIds = allocatedContainers.stream()
+ .map(x -> x.getNodeId()).collect(Collectors.toSet());
+ // Ensure unique nodes
+ Assert.assertEquals(4, nodeIds.size());
+ RejectedSchedulingRequest rej = rejectedReqs.get(0);
+ Assert.assertEquals(RejectionReason.COULD_NOT_PLACE_ON_NODE,
+ rej.getReason());
+ }
+
+ private static SchedulingRequest schedulingRequest(
+ int priority, long allocReqId, int cores, int mem, String... tags) {
+ return schedulingRequest(priority, allocReqId, cores, mem,
+ ExecutionType.GUARANTEED, tags);
+ }
+
+ private static SchedulingRequest schedulingRequest(
+ int priority, long allocReqId, int cores, int mem,
+ ExecutionType execType, String... tags) {
+ return SchedulingRequest.newBuilder()
+ .priority(Priority.newInstance(priority))
+ .allocationRequestId(allocReqId)
+ .allocationTags(new HashSet<>(Arrays.asList(tags)))
+ .executionType(ExecutionTypeRequest.newInstance(execType, true))
+ .resourceSizing(
+ ResourceSizing.newInstance(1, Resource.newInstance(mem, cores)))
+ .build();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/11] hadoop git commit: YARN-7682. Expose canSatisfyConstraints
utility function to validate a placement against a constraint. (Panagiotis
Garefalakis via asuresh)
Posted by as...@apache.org.
YARN-7682. Expose canSatisfyConstraints utility function to validate a placement against a constraint. (Panagiotis Garefalakis via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96186f5c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96186f5c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96186f5c
Branch: refs/heads/YARN-6592
Commit: 96186f5c3e28ebf9a8df9ee3706b2f1620fbef4b
Parents: 2bcd5cc
Author: Arun Suresh <as...@apache.org>
Authored: Wed Jan 3 08:00:50 2018 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:52:08 2018 -0800
----------------------------------------------------------------------
.../constraint/PlacementConstraintsUtil.java | 132 +++++++++
.../algorithm/DefaultPlacementAlgorithm.java | 55 +---
.../TestPlacementConstraintsUtil.java | 287 +++++++++++++++++++
.../constraint/TestPlacementProcessor.java | 204 +++++++++++--
4 files changed, 601 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96186f5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
new file mode 100644
index 0000000..956a3c9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import java.util.Iterator;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
+
+/**
+ * This class contains various static methods used by the Placement Algorithms
+ * to simplify constrained placement.
+ * (see also {@link DefaultPlacementAlgorithm}).
+ */
+@Public
+@Unstable
+public final class PlacementConstraintsUtil {
+
+ // Suppresses default constructor, ensuring non-instantiability.
+ private PlacementConstraintsUtil() {
+ }
+
+ /**
+ * Returns true if **single** application constraint with associated
+ * allocationTags and scope is satisfied by a specific scheduler Node.
+ *
+ * @param appId the application id
+ * @param sc the placement constraint
+ * @param te the target expression
+ * @param node the scheduler node
+ * @param tm the allocation tags store
+ * @return true if single application constraint is satisfied by node
+ * @throws InvalidAllocationTagsQueryException
+ */
+ private static boolean canSatisfySingleConstraintExpression(
+ ApplicationId appId, SingleConstraint sc, TargetExpression te,
+ SchedulerNode node, AllocationTagsManager tm)
+ throws InvalidAllocationTagsQueryException {
+ long minScopeCardinality = 0;
+ long maxScopeCardinality = 0;
+ if (sc.getScope() == PlacementConstraints.NODE) {
+ minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
+ te.getTargetValues(), Long::max);
+ maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
+ te.getTargetValues(), Long::min);
+ } else if (sc.getScope() == PlacementConstraints.RACK) {
+ minScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(), appId,
+ te.getTargetValues(), Long::max);
+ maxScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(), appId,
+ te.getTargetValues(), Long::min);
+ }
+ // Make sure Anti-affinity satisfies hard upper limit
+ maxScopeCardinality = sc.getMaxCardinality() == 0 ? maxScopeCardinality - 1
+ : maxScopeCardinality;
+
+ return (minScopeCardinality >= sc.getMinCardinality()
+ && maxScopeCardinality < sc.getMaxCardinality());
+ }
+
+ /**
+ * Returns true if all application constraints with associated allocationTags
+ * are **currently** satisfied by a specific scheduler Node.
+ * To do so the method retrieves and goes through all application constraint
+ * expressions and checks if the specific allocation is between the allowed
+ * min-max cardinality values under the constraint scope (Node/Rack/etc).
+ *
+ * @param appId the application id
+ * @param allocationTags the allocation tags set
+ * @param node the scheduler node
+ * @param pcm the placement constraints store
+ * @param tagsManager the allocation tags store
+ * @return true if all application constraints are satisfied by node
+ * @throws InvalidAllocationTagsQueryException
+ */
+ public static boolean canSatisfyConstraints(ApplicationId appId,
+ Set<String> allocationTags, SchedulerNode node,
+ PlacementConstraintManager pcm, AllocationTagsManager tagsManager)
+ throws InvalidAllocationTagsQueryException {
+ PlacementConstraint constraint = pcm.getConstraint(appId, allocationTags);
+ if (constraint == null) {
+ return true;
+ }
+ // Transform to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(constraint);
+ constraint = singleTransformer.transform();
+ AbstractConstraint sConstraintExpr = constraint.getConstraintExpr();
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ // Iterate through TargetExpressions
+ Iterator<TargetExpression> expIt = single.getTargetExpressions().iterator();
+ while (expIt.hasNext()) {
+ TargetExpression currentExp = expIt.next();
+ // Supporting AllocationTag Expressions for now
+ if (currentExp.getTargetType().equals(TargetType.ALLOCATION_TAG)) {
+ // Check if conditions are met
+ if (!canSatisfySingleConstraintExpression(appId, single, currentExp,
+ node, tagsManager)) {
+ return false;
+ }
+ }
+ }
+ // return true if all targetExpressions are satisfied
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96186f5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
index 395c156..9ed9ab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -19,19 +19,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algor
import java.util.Iterator;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutput;
@@ -65,58 +62,14 @@ public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm {
.getNodes(filter);
}
- /**
- * TODO: Method will be moved to PlacementConstraintsUtil class (YARN-7682)
- * @param applicationId
- * @param allocationTags
- * @param nodeId
- * @param tagsManager
- * @return boolean
- * @throws InvalidAllocationTagsQueryException
- */
- public boolean canAssign(ApplicationId applicationId,
- Set<String> allocationTags, NodeId nodeId,
- AllocationTagsManager tagsManager)
- throws InvalidAllocationTagsQueryException {
- PlacementConstraint constraint =
- constraintManager.getConstraint(applicationId, allocationTags);
- if (constraint == null) {
- return true;
- }
- // TODO: proper transformations
- // Currently works only for simple anti-affinity
- // NODE scope target expressions
- PlacementConstraintTransformations.SpecializedConstraintTransformer transformer =
- new PlacementConstraintTransformations.SpecializedConstraintTransformer(
- constraint);
- PlacementConstraint transform = transformer.transform();
- PlacementConstraint.TargetConstraint targetConstraint =
- (PlacementConstraint.TargetConstraint) transform.getConstraintExpr();
- // Assume a single target expression tag;
- // The Sample Algorithm assumes a constraint will always be a simple
- // Target Constraint with a single entry in the target set.
- // As mentioned in the class javadoc - This algorithm should be
- // used mostly for testing and validating end-2-end workflow.
- String targetTag = targetConstraint.getTargetExpressions().iterator().next()
- .getTargetValues().iterator().next();
- // TODO: Assuming anti-affinity constraint
- long nodeCardinality =
- tagsManager.getNodeCardinality(nodeId, applicationId, targetTag);
- if (nodeCardinality != 0) {
- return false;
- }
- // return true if it is a valid placement
- return true;
- }
-
public boolean attemptPlacementOnNode(ApplicationId appId,
SchedulingRequest schedulingRequest, SchedulerNode schedulerNode)
throws InvalidAllocationTagsQueryException {
int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
if (numAllocs > 0) {
- if (canAssign(appId,
- schedulingRequest.getAllocationTags(), schedulerNode.getNodeID(),
- tagsManager)) {
+ if (PlacementConstraintsUtil.canSatisfyConstraints(appId,
+ schedulingRequest.getAllocationTags(), schedulerNode,
+ constraintManager, tagsManager)) {
return true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96186f5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java
new file mode 100644
index 0000000..7492233
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintsUtil.java
@@ -0,0 +1,287 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+import java.util.AbstractMap;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Test the PlacementConstraint Utility class functionality.
+ */
+public class TestPlacementConstraintsUtil {
+
+ private List<RMNode> rmNodes;
+ private RMContext rmContext;
+ private static final int GB = 1024;
+ private ApplicationId appId1;
+ private PlacementConstraint c1, c2, c3, c4;
+ private Set<String> sourceTag1, sourceTag2;
+ private Map<Set<String>, PlacementConstraint> constraintMap1, constraintMap2;
+
+ @Before
+ public void setup() {
+ MockRM rm = new MockRM();
+ rm.start();
+ MockNodes.resetHostIds();
+ rmNodes = MockNodes.newNodes(2, 2, Resource.newInstance(4096, 4));
+ for (RMNode rmNode : rmNodes) {
+ rm.getRMContext().getRMNodes().putIfAbsent(rmNode.getNodeID(), rmNode);
+ }
+ rmContext = rm.getRMContext();
+
+ // Build appIDs, constraints, source tags, and constraint map.
+ long ts = System.currentTimeMillis();
+ appId1 = BuilderUtils.newApplicationId(ts, 123);
+
+ c1 = PlacementConstraints.build(targetIn(NODE, allocationTag("hbase-m")));
+ c2 = PlacementConstraints.build(targetIn(RACK, allocationTag("hbase-rs")));
+ c3 = PlacementConstraints
+ .build(targetNotIn(NODE, allocationTag("hbase-m")));
+ c4 = PlacementConstraints
+ .build(targetNotIn(RACK, allocationTag("hbase-rs")));
+
+ sourceTag1 = new HashSet<>(Arrays.asList("spark"));
+ sourceTag2 = new HashSet<>(Arrays.asList("zk"));
+
+ constraintMap1 = Stream
+ .of(new AbstractMap.SimpleEntry<>(sourceTag1, c1),
+ new AbstractMap.SimpleEntry<>(sourceTag2, c2))
+ .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey,
+ AbstractMap.SimpleEntry::getValue));
+ constraintMap2 = Stream
+ .of(new AbstractMap.SimpleEntry<>(sourceTag1, c3),
+ new AbstractMap.SimpleEntry<>(sourceTag2, c4))
+ .collect(Collectors.toMap(AbstractMap.SimpleEntry::getKey,
+ AbstractMap.SimpleEntry::getValue));
+ }
+
+ @Test
+ public void testNodeAffinityAssignment()
+ throws InvalidAllocationTagsQueryException {
+ PlacementConstraintManagerService pcm =
+ new MemoryPlacementConstraintManager();
+ AllocationTagsManager tm = new AllocationTagsManager(rmContext);
+ // Register App1 with affinity constraint map
+ pcm.registerApplication(appId1, constraintMap1);
+ // No containers are running so all 'zk' and 'spark' allocations should fail
+ // on every cluster NODE
+ Iterator<RMNode> nodeIterator = rmNodes.iterator();
+ while (nodeIterator.hasNext()) {
+ RMNode currentNode = nodeIterator.next();
+ FiCaSchedulerNode schedulerNode = TestUtils.getMockNode(
+ currentNode.getHostName(), currentNode.getRackName(), 123, 4 * GB);
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode, pcm, tm));
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode, pcm, tm));
+ }
+ /**
+ * Now place container:
+ * Node0:123 (Rack1):
+ * container_app1_1 (hbase-m)
+ */
+ RMNode n0_r1 = rmNodes.get(0);
+ RMNode n1_r1 = rmNodes.get(1);
+ RMNode n2_r2 = rmNodes.get(2);
+ RMNode n3_r2 = rmNodes.get(3);
+ FiCaSchedulerNode schedulerNode0 = TestUtils
+ .getMockNode(n0_r1.getHostName(), n0_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode1 = TestUtils
+ .getMockNode(n1_r1.getHostName(), n1_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode2 = TestUtils
+ .getMockNode(n2_r2.getHostName(), n2_r2.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode3 = TestUtils
+ .getMockNode(n3_r2.getHostName(), n3_r2.getRackName(), 123, 4 * GB);
+ // 1 Containers on node 0 with allocationTag 'hbase-m'
+ ContainerId hbase_m = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
+ tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-m"));
+
+ // 'spark' placement on Node0 should now SUCCEED
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode0, pcm, tm));
+ // FAIL on the rest of the nodes
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode1, pcm, tm));
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode2, pcm, tm));
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode3, pcm, tm));
+ }
+
+ @Test
+ public void testRackAffinityAssignment()
+ throws InvalidAllocationTagsQueryException {
+ PlacementConstraintManagerService pcm =
+ new MemoryPlacementConstraintManager();
+ AllocationTagsManager tm = new AllocationTagsManager(rmContext);
+ // Register App1 with affinity constraint map
+ pcm.registerApplication(appId1, constraintMap1);
+ /**
+ * Now place container:
+ * Node0:123 (Rack1):
+ * container_app1_1 (hbase-rs)
+ */
+ RMNode n0_r1 = rmNodes.get(0);
+ RMNode n1_r1 = rmNodes.get(1);
+ RMNode n2_r2 = rmNodes.get(2);
+ RMNode n3_r2 = rmNodes.get(3);
+ // 1 Containers on Node0-Rack1 with allocationTag 'hbase-rs'
+ ContainerId hbase_m = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
+ tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-rs"));
+
+ FiCaSchedulerNode schedulerNode0 = TestUtils
+ .getMockNode(n0_r1.getHostName(), n0_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode1 = TestUtils
+ .getMockNode(n1_r1.getHostName(), n1_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode2 = TestUtils
+ .getMockNode(n2_r2.getHostName(), n2_r2.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode3 = TestUtils
+ .getMockNode(n3_r2.getHostName(), n3_r2.getRackName(), 123, 4 * GB);
+ // 'zk' placement on Rack1 should now SUCCEED
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode0, pcm, tm));
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode1, pcm, tm));
+
+ // FAIL on the rest of the RACKs
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode2, pcm, tm));
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode3, pcm, tm));
+ }
+
+ @Test
+ public void testNodeAntiAffinityAssignment()
+ throws InvalidAllocationTagsQueryException {
+ PlacementConstraintManagerService pcm =
+ new MemoryPlacementConstraintManager();
+ AllocationTagsManager tm = new AllocationTagsManager(rmContext);
+ // Register App1 with anti-affinity constraint map
+ pcm.registerApplication(appId1, constraintMap2);
+ /**
+ * place container:
+ * Node0:123 (Rack1):
+ * container_app1_1 (hbase-m)
+ */
+ RMNode n0_r1 = rmNodes.get(0);
+ RMNode n1_r1 = rmNodes.get(1);
+ RMNode n2_r2 = rmNodes.get(2);
+ RMNode n3_r2 = rmNodes.get(3);
+ FiCaSchedulerNode schedulerNode0 = TestUtils
+ .getMockNode(n0_r1.getHostName(), n0_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode1 = TestUtils
+ .getMockNode(n1_r1.getHostName(), n1_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode2 = TestUtils
+ .getMockNode(n2_r2.getHostName(), n2_r2.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode3 = TestUtils
+ .getMockNode(n3_r2.getHostName(), n3_r2.getRackName(), 123, 4 * GB);
+ // 1 Containers on node 0 with allocationTag 'hbase-m'
+ ContainerId hbase_m = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
+ tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-m"));
+
+ // 'spark' placement on Node0 should now FAIL
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode0, pcm, tm));
+ // SUCCEED on the rest of the nodes
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode1, pcm, tm));
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode2, pcm, tm));
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag1, schedulerNode3, pcm, tm));
+ }
+
+ @Test
+ public void testRackAntiAffinityAssignment()
+ throws InvalidAllocationTagsQueryException {
+ AllocationTagsManager tm = new AllocationTagsManager(rmContext);
+ PlacementConstraintManagerService pcm =
+ new MemoryPlacementConstraintManager();
+ // Register App1 with anti-affinity constraint map
+ pcm.registerApplication(appId1, constraintMap2);
+ /**
+ * Place container:
+ * Node0:123 (Rack1):
+ * container_app1_1 (hbase-rs)
+ */
+ RMNode n0_r1 = rmNodes.get(0);
+ RMNode n1_r1 = rmNodes.get(1);
+ RMNode n2_r2 = rmNodes.get(2);
+ RMNode n3_r2 = rmNodes.get(3);
+ // 1 Containers on Node0-Rack1 with allocationTag 'hbase-rs'
+ ContainerId hbase_m = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(appId1, 0), 0);
+ tm.addContainer(n0_r1.getNodeID(), hbase_m, ImmutableSet.of("hbase-rs"));
+
+ FiCaSchedulerNode schedulerNode0 = TestUtils
+ .getMockNode(n0_r1.getHostName(), n0_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode1 = TestUtils
+ .getMockNode(n1_r1.getHostName(), n1_r1.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode2 = TestUtils
+ .getMockNode(n2_r2.getHostName(), n2_r2.getRackName(), 123, 4 * GB);
+ FiCaSchedulerNode schedulerNode3 = TestUtils
+ .getMockNode(n3_r2.getHostName(), n3_r2.getRackName(), 123, 4 * GB);
+
+ // 'zk' placement on Rack1 should FAIL
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode0, pcm, tm));
+ Assert.assertFalse(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode1, pcm, tm));
+
+ // SUCCEED on the rest of the RACKs
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode2, pcm, tm));
+ Assert.assertTrue(PlacementConstraintsUtil.canSatisfyConstraints(appId1,
+ sourceTag2, schedulerNode3, pcm, tm));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96186f5c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
index 87dd5b7..c260fe0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.RejectionReason;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceSizing;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -48,16 +49,21 @@ import org.junit.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import static java.lang.Thread.sleep;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn;
/**
* This tests end2end workflow of the constraint placement framework.
@@ -104,7 +110,7 @@ public class TestPlacementProcessor {
}
@Test(timeout = 300000)
- public void testPlacement() throws Exception {
+ public void testAntiAffinityPlacement() throws Exception {
HashMap<NodeId, MockNM> nodes = new HashMap<>();
MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
nodes.put(nm1.getNodeId(), nm1);
@@ -120,44 +126,174 @@ public class TestPlacementProcessor {
nm4.registerNode();
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ // Containers with allocationTag 'foo' are restricted to 1 per NODE
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
- Collections.singletonMap(
- Collections.singleton("foo"),
+ Collections.singletonMap(Collections.singleton("foo"),
PlacementConstraints.build(
- PlacementConstraints.targetNotIn(NODE, allocationTag("foo")))
- ));
+ PlacementConstraints.targetNotIn(NODE, allocationTag("foo")))));
am1.addSchedulingRequest(
- Arrays.asList(
- schedulingRequest(1, 1, 1, 512, "foo"),
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "foo"),
schedulingRequest(1, 2, 1, 512, "foo"),
schedulingRequest(1, 3, 1, 512, "foo"),
- schedulingRequest(1, 5, 1, 512, "foo"))
- );
+ schedulingRequest(1, 5, 1, 512, "foo")));
AllocateResponse allocResponse = am1.schedule(); // send the request
List<Container> allocatedContainers = new ArrayList<>();
allocatedContainers.addAll(allocResponse.getAllocatedContainers());
// kick the scheduler
-
- while (allocatedContainers.size() < 4) {
- nm1.nodeHeartbeat(true);
- nm2.nodeHeartbeat(true);
- nm3.nodeHeartbeat(true);
- nm4.nodeHeartbeat(true);
- LOG.info("Waiting for containers to be created for app 1...");
- sleep(1000);
- allocResponse = am1.schedule();
- allocatedContainers.addAll(allocResponse.getAllocatedContainers());
- }
+ waitForContainerAllocation(nodes.values(), am1, allocatedContainers, 4);
Assert.assertEquals(4, allocatedContainers.size());
- Set<NodeId> nodeIds = allocatedContainers.stream()
- .map(x -> x.getNodeId()).collect(Collectors.toSet());
- // Ensure unique nodes
+ Set<NodeId> nodeIds = allocatedContainers.stream().map(x -> x.getNodeId())
+ .collect(Collectors.toSet());
+ // Ensure unique nodes (antiaffinity)
Assert.assertEquals(4, nodeIds.size());
}
@Test(timeout = 300000)
+ public void testCardinalityPlacement() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ // Containers with allocationTag 'foo' should not exceed 4 per NODE
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
+ Collections.singletonMap(Collections.singleton("foo"),
+ PlacementConstraints.build(PlacementConstraints
+ .targetCardinality(NODE, 0, 4, allocationTag("foo")))));
+ am1.addSchedulingRequest(
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 4, 1, 512, "foo"),
+ schedulingRequest(1, 5, 1, 512, "foo"),
+ schedulingRequest(1, 6, 1, 512, "foo"),
+ schedulingRequest(1, 7, 1, 512, "foo"),
+ schedulingRequest(1, 8, 1, 512, "foo")));
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+
+ // kick the scheduler
+ waitForContainerAllocation(nodes.values(), am1, allocatedContainers, 8);
+
+ Assert.assertEquals(8, allocatedContainers.size());
+ Map<NodeId, Long> nodeIdContainerIdMap =
+ allocatedContainers.stream().collect(
+ Collectors.groupingBy(c -> c.getNodeId(), Collectors.counting()));
+ // Ensure no more than 4 containers per node
+ for (NodeId n : nodeIdContainerIdMap.keySet()) {
+ Assert.assertTrue(nodeIdContainerIdMap.get(n) < 5);
+ }
+ }
+
+ @Test(timeout = 300000)
+ public void testAffinityPlacement() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ // Containers with allocationTag 'foo' should be placed where
+ // containers with allocationTag 'bar' are already running
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
+ Collections.singletonMap(Collections.singleton("foo"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetIn(NODE, allocationTag("bar")))));
+ am1.addSchedulingRequest(
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "bar"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 4, 1, 512, "foo"),
+ schedulingRequest(1, 5, 1, 512, "foo")));
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+
+ // kick the scheduler
+ waitForContainerAllocation(nodes.values(), am1, allocatedContainers, 5);
+
+ Assert.assertEquals(5, allocatedContainers.size());
+ Set<NodeId> nodeIds = allocatedContainers.stream().map(x -> x.getNodeId())
+ .collect(Collectors.toSet());
+ // Ensure all containers end up on the same node (affinity)
+ Assert.assertEquals(1, nodeIds.size());
+ }
+
+ @Test(timeout = 300000)
+ public void testComplexPlacement() throws Exception {
+ HashMap<NodeId, MockNM> nodes = new HashMap<>();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm1.getNodeId(), nm1);
+ MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm2.getNodeId(), nm2);
+ MockNM nm3 = new MockNM("h3:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm3.getNodeId(), nm3);
+ MockNM nm4 = new MockNM("h4:1234", 4096, rm.getResourceTrackerService());
+ nodes.put(nm4.getNodeId(), nm4);
+ nm1.registerNode();
+ nm2.registerNode();
+ nm3.registerNode();
+ nm4.registerNode();
+
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ Map<Set<String>, PlacementConstraint> constraintMap = new HashMap<>();
+ // Containers with allocationTag 'bar' should not exceed 1 per NODE
+ constraintMap.put(Collections.singleton("bar"),
+ PlacementConstraints.build(targetNotIn(NODE, allocationTag("bar"))));
+ // Containers with allocationTag 'foo' should be placed where 'bar' exists
+ constraintMap.put(Collections.singleton("foo"),
+ PlacementConstraints.build(targetIn(NODE, allocationTag("bar"))));
+ // Containers with allocationTag 'foo' should not exceed 2 per NODE
+ constraintMap.put(Collections.singleton("foo"), PlacementConstraints
+ .build(targetCardinality(NODE, 0, 2, allocationTag("foo"))));
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2, constraintMap);
+ am1.addSchedulingRequest(
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "bar"),
+ schedulingRequest(1, 2, 1, 512, "bar"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 4, 1, 512, "foo"),
+ schedulingRequest(1, 5, 1, 512, "foo"),
+ schedulingRequest(1, 6, 1, 512, "foo")));
+ AllocateResponse allocResponse = am1.schedule(); // send the request
+ List<Container> allocatedContainers = new ArrayList<>();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+
+ // kick the scheduler
+ waitForContainerAllocation(nodes.values(), am1, allocatedContainers, 6);
+
+ Assert.assertEquals(6, allocatedContainers.size());
+ Map<NodeId, Long> nodeIdContainerIdMap =
+ allocatedContainers.stream().collect(
+ Collectors.groupingBy(c -> c.getNodeId(), Collectors.counting()));
+ // Ensure no more than 3 containers per node (1 'bar', 2 'foo')
+ for (NodeId n : nodeIdContainerIdMap.keySet()) {
+ Assert.assertTrue(nodeIdContainerIdMap.get(n) < 4);
+ }
+ }
+
+ @Test(timeout = 300000)
public void testSchedulerRejection() throws Exception {
HashMap<NodeId, MockNM> nodes = new HashMap<>();
MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
@@ -174,6 +310,7 @@ public class TestPlacementProcessor {
nm4.registerNode();
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ // Containers with allocationTag 'foo' are restricted to 1 per NODE
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
Collections.singletonMap(
Collections.singleton("foo"),
@@ -196,7 +333,6 @@ public class TestPlacementProcessor {
rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
// kick the scheduler
-
while (allocCount < 11) {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
@@ -253,9 +389,10 @@ public class TestPlacementProcessor {
nm2.registerNode();
nm3.registerNode();
nm4.registerNode();
- // No not register nm5 yet..
+ // Do not register nm5 yet..
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ // Containers with allocationTag 'foo' are restricted to 1 per NODE
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
Collections.singletonMap(
Collections.singleton("foo"),
@@ -323,6 +460,7 @@ public class TestPlacementProcessor {
nm4.registerNode();
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ // Containers with allocationTag 'foo' are restricted to 1 per NODE
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2,
Collections.singletonMap(
Collections.singleton("foo"),
@@ -346,7 +484,6 @@ public class TestPlacementProcessor {
rejectedReqs.addAll(allocResponse.getRejectedSchedulingRequests());
// kick the scheduler
-
while (allocCount < 11) {
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(true);
@@ -373,6 +510,21 @@ public class TestPlacementProcessor {
rej.getReason());
}
+ private static void waitForContainerAllocation(Collection<MockNM> nodes,
+ MockAM am, List<Container> allocatedContainers, int containerNum)
+ throws Exception {
+ while (allocatedContainers.size() < containerNum) {
+ for (MockNM node : nodes) {
+ node.nodeHeartbeat(true);
+ }
+ LOG.info("Waiting for containers to be created for "
+ + am.getApplicationAttemptId().getApplicationId() + "...");
+ sleep(1000);
+ AllocateResponse allocResponse = am.schedule();
+ allocatedContainers.addAll(allocResponse.getAllocatedContainers());
+ }
+ }
+
protected static SchedulingRequest schedulingRequest(
int priority, long allocReqId, int cores, int mem, String... tags) {
return schedulingRequest(priority, allocReqId, cores, mem,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/11] hadoop git commit: YARN-7613. Implement Basic algorithm for
constraint based placement. (Panagiotis Garefalakis via asuresh)
Posted by as...@apache.org.
YARN-7613. Implement Basic algorithm for constraint based placement. (Panagiotis Garefalakis via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bcd5cc1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bcd5cc1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bcd5cc1
Branch: refs/heads/YARN-6592
Commit: 2bcd5cc1aaaa2430fce0adf93828c5f520d56780
Parents: e802a6c
Author: Arun Suresh <as...@apache.org>
Authored: Wed Dec 27 22:59:22 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:51:47 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 4 +
.../src/main/resources/yarn-default.xml | 8 +-
.../rmcontainer/RMContainerImpl.java | 10 +-
.../constraint/AllocationTagsManager.java | 121 ++++++++++---
.../algorithm/DefaultPlacementAlgorithm.java | 172 +++++++++++++++++++
.../iterators/PopularTagsIterator.java | 71 ++++++++
.../algorithm/iterators/SerialIterator.java | 53 ++++++
.../algorithm/iterators/package-info.java | 29 ++++
.../constraint/algorithm/package-info.java | 29 ++++
.../constraint/processor/BatchedRequests.java | 45 ++++-
.../processor/PlacementProcessor.java | 32 ++--
.../processor/SamplePlacementAlgorithm.java | 144 ----------------
.../constraint/TestAllocationTagsManager.java | 156 ++++++++++++-----
.../TestBatchedRequestsIterators.java | 82 +++++++++
.../constraint/TestPlacementProcessor.java | 4 +-
15 files changed, 721 insertions(+), 239 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 03c24d4..af83d8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -534,6 +534,10 @@ public class YarnConfiguration extends Configuration {
public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_CLASS =
RM_PREFIX + "placement-constraints.algorithm.class";
+ /** Used for BasicPlacementAlgorithm - default SERIAL. **/
+ public static final String RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR =
+ RM_PREFIX + "placement-constraints.algorithm.iterator";
+
public static final String RM_PLACEMENT_CONSTRAINTS_ENABLED =
RM_PREFIX + "placement-constraints.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0285069..62bbdb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -145,7 +145,13 @@
<property>
<description>Constraint Placement Algorithm to be used.</description>
<name>yarn.resourcemanager.placement-constraints.algorithm.class</name>
- <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.SamplePlacementAlgorithm</value>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm</value>
+ </property>
+
+ <property>
+ <description>Placement Algorithm Requests Iterator to be used.</description>
+ <name>yarn.resourcemanager.placement-constraints.algorithm.iterator</name>
+ <value>SERIAL</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index c873509..2c4ef7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -579,9 +579,8 @@ public class RMContainerImpl implements RMContainer {
public void transition(RMContainerImpl container, RMContainerEvent event) {
// Notify placementManager
container.rmContext.getAllocationTagsManager().addContainer(
- container.getNodeId(),
- container.getApplicationAttemptId().getApplicationId(),
- container.getContainerId(), container.getAllocationTags());
+ container.getNodeId(), container.getContainerId(),
+ container.getAllocationTags());
container.eventHandler.handle(new RMAppAttemptEvent(
container.appAttemptId, RMAppAttemptEventType.CONTAINER_ALLOCATED));
@@ -696,9 +695,8 @@ public class RMContainerImpl implements RMContainer {
public void transition(RMContainerImpl container, RMContainerEvent event) {
// Notify placementManager
container.rmContext.getAllocationTagsManager().removeContainer(
- container.getNodeId(),
- container.getApplicationAttemptId().getApplicationId(),
- container.getContainerId(), container.getAllocationTags());
+ container.getNodeId(), container.getContainerId(),
+ container.getAllocationTags());
RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index 7b0b959..4bb3e79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -24,6 +24,7 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -54,24 +55,27 @@ public class AllocationTagsManager {
private final RMContext rmContext;
// Application's tags to Node
- private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings =
+ private Map<ApplicationId, TypeToCountedTags> perAppNodeMappings =
new HashMap<>();
// Application's tags to Rack
- private Map<ApplicationId, NodeToCountedTags> perAppRackMappings =
+ private Map<ApplicationId, TypeToCountedTags> perAppRackMappings =
new HashMap<>();
+ // Application's Temporary containers mapping
+ private Map<ApplicationId, Map<NodeId, Map<ContainerId, Set<String>>>>
+ appTempMappings = new HashMap<>();
// Global tags to node mapping (used to fast return aggregated tags
// cardinality across apps)
- private NodeToCountedTags<NodeId> globalNodeMapping = new NodeToCountedTags();
+ private TypeToCountedTags<NodeId> globalNodeMapping = new TypeToCountedTags();
// Global tags to Rack mapping
- private NodeToCountedTags<String> globalRackMapping = new NodeToCountedTags();
+ private TypeToCountedTags<String> globalRackMapping = new TypeToCountedTags();
/**
* Generic store mapping type <T> to counted tags.
* Currently used both for NodeId to Tag, Count and Rack to Tag, Count
*/
@VisibleForTesting
- static class NodeToCountedTags<T> {
+ static class TypeToCountedTags<T> {
// Map<Type, Map<Tag, Count>>
private Map<T, Map<String, Long>> typeToTagsWithCount = new HashMap<>();
@@ -209,25 +213,31 @@ public class AllocationTagsManager {
}
@VisibleForTesting
- Map<ApplicationId, NodeToCountedTags> getPerAppNodeMappings() {
+ Map<ApplicationId, TypeToCountedTags> getPerAppNodeMappings() {
return perAppNodeMappings;
}
@VisibleForTesting
- Map<ApplicationId, NodeToCountedTags> getPerAppRackMappings() {
+ Map<ApplicationId, TypeToCountedTags> getPerAppRackMappings() {
return perAppRackMappings;
}
@VisibleForTesting
- NodeToCountedTags getGlobalNodeMapping() {
+ TypeToCountedTags getGlobalNodeMapping() {
return globalNodeMapping;
}
@VisibleForTesting
- NodeToCountedTags getGlobalRackMapping() {
+ TypeToCountedTags getGlobalRackMapping() {
return globalRackMapping;
}
+ @VisibleForTesting
+ public Map<NodeId, Map<ContainerId, Set<String>>> getAppTempMappings(
+ ApplicationId applicationId) {
+ return appTempMappings.get(applicationId);
+ }
+
public AllocationTagsManager(RMContext context) {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
@@ -235,18 +245,52 @@ public class AllocationTagsManager {
rmContext = context;
}
+ //
+
+ /**
+ * Method adds a temporary fake-container tag to Node mapping.
+ * Used by the constrained placement algorithm to keep track of containers
+ * that are currently placed on nodes but are not yet allocated.
+ * @param nodeId
+ * @param applicationId
+ * @param allocationTags
+ */
+ public void addTempContainer(NodeId nodeId, ApplicationId applicationId,
+ Set<String> allocationTags) {
+ ContainerId tmpContainer = ContainerId.newContainerId(
+ ApplicationAttemptId.newInstance(applicationId, 1), System.nanoTime());
+
+ writeLock.lock();
+ try {
+ Map<NodeId, Map<ContainerId, Set<String>>> appTempMapping =
+ appTempMappings.computeIfAbsent(applicationId, k -> new HashMap<>());
+ Map<ContainerId, Set<String>> containerTempMapping =
+ appTempMapping.computeIfAbsent(nodeId, k -> new HashMap<>());
+ containerTempMapping.put(tmpContainer, allocationTags);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Added TEMP container=" + tmpContainer + " with tags=["
+ + StringUtils.join(allocationTags, ",") + "]");
+ }
+ } finally {
+ writeLock.unlock();
+ }
+
+ addContainer(nodeId, tmpContainer, allocationTags);
+ }
+
/**
* Notify container allocated on a node.
*
* @param nodeId allocated node.
- * @param applicationId applicationId
* @param containerId container id.
* @param allocationTags allocation tags, see
* {@link SchedulingRequest#getAllocationTags()}
* application_id will be added to allocationTags.
*/
- public void addContainer(NodeId nodeId, ApplicationId applicationId,
- ContainerId containerId, Set<String> allocationTags) {
+ public void addContainer(NodeId nodeId, ContainerId containerId,
+ Set<String> allocationTags) {
+ ApplicationId applicationId =
+ containerId.getApplicationAttemptId().getApplicationId();
String applicationIdTag =
AllocationTagsNamespaces.APP_ID + applicationId.toString();
@@ -260,10 +304,10 @@ public class AllocationTagsManager {
writeLock.lock();
try {
- NodeToCountedTags perAppTagsMapping = perAppNodeMappings
- .computeIfAbsent(applicationId, k -> new NodeToCountedTags());
- NodeToCountedTags perAppRackTagsMapping = perAppRackMappings
- .computeIfAbsent(applicationId, k -> new NodeToCountedTags());
+ TypeToCountedTags perAppTagsMapping = perAppNodeMappings
+ .computeIfAbsent(applicationId, k -> new TypeToCountedTags());
+ TypeToCountedTags perAppRackTagsMapping = perAppRackMappings
+ .computeIfAbsent(applicationId, k -> new TypeToCountedTags());
// Covering test-cases where context is mocked
String nodeRack = (rmContext.getRMNodes() != null
&& rmContext.getRMNodes().get(nodeId) != null)
@@ -294,12 +338,13 @@ public class AllocationTagsManager {
* Notify container removed.
*
* @param nodeId nodeId
- * @param applicationId applicationId
* @param containerId containerId.
* @param allocationTags allocation tags for given container
*/
- public void removeContainer(NodeId nodeId, ApplicationId applicationId,
+ public void removeContainer(NodeId nodeId,
ContainerId containerId, Set<String> allocationTags) {
+ ApplicationId applicationId =
+ containerId.getApplicationAttemptId().getApplicationId();
String applicationIdTag =
AllocationTagsNamespaces.APP_ID + applicationId.toString();
boolean useSet = false;
@@ -313,9 +358,9 @@ public class AllocationTagsManager {
writeLock.lock();
try {
- NodeToCountedTags perAppTagsMapping =
+ TypeToCountedTags perAppTagsMapping =
perAppNodeMappings.get(applicationId);
- NodeToCountedTags perAppRackTagsMapping =
+ TypeToCountedTags perAppRackTagsMapping =
perAppRackMappings.get(applicationId);
if (perAppTagsMapping == null) {
return;
@@ -354,6 +399,34 @@ public class AllocationTagsManager {
}
/**
+ * Method removes temporary containers associated with an application
+ * Used by the placement algorithm to clean temporary tags at the end of
+ * a placement cycle.
+ * @param applicationId Application Id.
+ */
+ public void cleanTempContainers(ApplicationId applicationId) {
+
+ if (!appTempMappings.get(applicationId).isEmpty()) {
+ appTempMappings.get(applicationId).entrySet().stream().forEach(nodeE -> {
+ nodeE.getValue().entrySet().stream().forEach(containerE -> {
+ removeContainer(nodeE.getKey(), containerE.getKey(),
+ containerE.getValue());
+ });
+ });
+ writeLock.lock();
+ try {
+ appTempMappings.remove(applicationId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removed TEMP containers of app=" + applicationId);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+ }
+
+
+ /**
* Get Node cardinality for a specific tag.
* When applicationId is null, method returns aggregated cardinality
*
@@ -378,7 +451,7 @@ public class AllocationTagsManager {
"Must specify nodeId/tag to query cardinality");
}
- NodeToCountedTags mapping;
+ TypeToCountedTags mapping;
if (applicationId != null) {
mapping = perAppNodeMappings.get(applicationId);
} else {
@@ -419,7 +492,7 @@ public class AllocationTagsManager {
"Must specify rack/tag to query cardinality");
}
- NodeToCountedTags mapping;
+ TypeToCountedTags mapping;
if (applicationId != null) {
mapping = perAppRackMappings.get(applicationId);
} else {
@@ -492,7 +565,7 @@ public class AllocationTagsManager {
"Must specify nodeId/tags/op to query cardinality");
}
- NodeToCountedTags mapping;
+ TypeToCountedTags mapping;
if (applicationId != null) {
mapping = perAppNodeMappings.get(applicationId);
} else {
@@ -540,7 +613,7 @@ public class AllocationTagsManager {
"Must specify rack/tags/op to query cardinality");
}
- NodeToCountedTags mapping;
+ TypeToCountedTags mapping;
if (applicationId != null) {
mapping = perAppRackMappings.get(applicationId);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
new file mode 100644
index 0000000..395c156
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/DefaultPlacementAlgorithm.java
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutput;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutputCollector;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.BatchedRequests;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.NodeCandidateSelector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Basic placement algorithm.
+ * Supports different Iterators at SchedulingRequest level including:
+ * Serial, PopularTags
+ */
+public class DefaultPlacementAlgorithm implements ConstraintPlacementAlgorithm {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultPlacementAlgorithm.class);
+
+ private AllocationTagsManager tagsManager;
+ private PlacementConstraintManager constraintManager;
+ private NodeCandidateSelector nodeSelector;
+
+ @Override
+ public void init(RMContext rmContext) {
+ this.tagsManager = rmContext.getAllocationTagsManager();
+ this.constraintManager = rmContext.getPlacementConstraintManager();
+ this.nodeSelector =
+ filter -> ((AbstractYarnScheduler) (rmContext).getScheduler())
+ .getNodes(filter);
+ }
+
+ /**
+ * TODO: Method will be moved to PlacementConstraintsUtil class (YARN-7682)
+ * @param applicationId
+ * @param allocationTags
+ * @param nodeId
+ * @param tagsManager
+ * @return boolean
+ * @throws InvalidAllocationTagsQueryException
+ */
+ public boolean canAssign(ApplicationId applicationId,
+ Set<String> allocationTags, NodeId nodeId,
+ AllocationTagsManager tagsManager)
+ throws InvalidAllocationTagsQueryException {
+ PlacementConstraint constraint =
+ constraintManager.getConstraint(applicationId, allocationTags);
+ if (constraint == null) {
+ return true;
+ }
+ // TODO: proper transformations
+ // Currently works only for simple anti-affinity
+ // NODE scope target expressions
+ PlacementConstraintTransformations.SpecializedConstraintTransformer transformer =
+ new PlacementConstraintTransformations.SpecializedConstraintTransformer(
+ constraint);
+ PlacementConstraint transform = transformer.transform();
+ PlacementConstraint.TargetConstraint targetConstraint =
+ (PlacementConstraint.TargetConstraint) transform.getConstraintExpr();
+ // Assume a single target expression tag;
+ // The Sample Algorithm assumes a constraint will always be a simple
+ // Target Constraint with a single entry in the target set.
+ // As mentioned in the class javadoc - This algorithm should be
+ // used mostly for testing and validating end-2-end workflow.
+ String targetTag = targetConstraint.getTargetExpressions().iterator().next()
+ .getTargetValues().iterator().next();
+ // TODO: Assuming anti-affinity constraint
+ long nodeCardinality =
+ tagsManager.getNodeCardinality(nodeId, applicationId, targetTag);
+ if (nodeCardinality != 0) {
+ return false;
+ }
+ // return true if it is a valid placement
+ return true;
+ }
+
+ public boolean attemptPlacementOnNode(ApplicationId appId,
+ SchedulingRequest schedulingRequest, SchedulerNode schedulerNode)
+ throws InvalidAllocationTagsQueryException {
+ int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
+ if (numAllocs > 0) {
+ if (canAssign(appId,
+ schedulingRequest.getAllocationTags(), schedulerNode.getNodeID(),
+ tagsManager)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ @Override
+ public void place(ConstraintPlacementAlgorithmInput input,
+ ConstraintPlacementAlgorithmOutputCollector collector) {
+ BatchedRequests requests = (BatchedRequests) input;
+ ConstraintPlacementAlgorithmOutput resp =
+ new ConstraintPlacementAlgorithmOutput(requests.getApplicationId());
+ List<SchedulerNode> allNodes = nodeSelector.selectNodes(null);
+
+ Iterator<SchedulingRequest> requestIterator = requests.iterator();
+ while (requestIterator.hasNext()) {
+ SchedulingRequest schedulingRequest = requestIterator.next();
+ Iterator<SchedulerNode> nodeIter = allNodes.iterator();
+ int numAllocs = schedulingRequest.getResourceSizing().getNumAllocations();
+ while (nodeIter.hasNext() && numAllocs > 0) {
+ SchedulerNode node = nodeIter.next();
+ try {
+ if (attemptPlacementOnNode(requests.getApplicationId(),
+ schedulingRequest, node)) {
+ schedulingRequest.getResourceSizing()
+ .setNumAllocations(--numAllocs);
+ PlacedSchedulingRequest placedReq =
+ new PlacedSchedulingRequest(schedulingRequest);
+ placedReq.setPlacementAttempt(requests.getPlacementAttempt());
+ placedReq.getNodes().add(node);
+ resp.getPlacedRequests().add(placedReq);
+ numAllocs =
+ schedulingRequest.getResourceSizing().getNumAllocations();
+ // Add temp-container tags for current placement cycle
+ this.tagsManager.addTempContainer(node.getNodeID(),
+ requests.getApplicationId(),
+ schedulingRequest.getAllocationTags());
+ }
+ } catch (InvalidAllocationTagsQueryException e) {
+ LOG.warn("Got exception from TagManager !", e);
+ }
+ }
+ }
+ // Add all requests whose numAllocations still > 0 to rejected list.
+ requests.getSchedulingRequests().stream()
+ .filter(sReq -> sReq.getResourceSizing().getNumAllocations() > 0)
+ .forEach(rejReq -> resp.getRejectedRequests().add(rejReq));
+ collector.collect(resp);
+ // Clean current temp-container tags
+ this.tagsManager.cleanTempContainers(requests.getApplicationId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/PopularTagsIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/PopularTagsIterator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/PopularTagsIterator.java
new file mode 100644
index 0000000..ca3e351
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/PopularTagsIterator.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.iterators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+/**
+ * Traverse Scheduling requests with the most popular tags (count) first.
+ * Currently the count is per Batch but could use TagManager for global count.
+ */
+public class PopularTagsIterator implements Iterator<SchedulingRequest> {
+
+ private final List<SchedulingRequest> schedulingRequestList;
+ private int cursor;
+
+ public PopularTagsIterator(Collection<SchedulingRequest> schedulingRequests) {
+ this.schedulingRequestList = new ArrayList<>(schedulingRequests);
+ // Most popular First
+ Collections.sort(schedulingRequestList,
+ (o1, o2) -> (int) getTagPopularity(o2) - (int) getTagPopularity(o1));
+
+ this.cursor = 0;
+ }
+
+ private long getTagPopularity(SchedulingRequest o1) {
+ long max = 0;
+ for (String tag : o1.getAllocationTags()) {
+ long count = schedulingRequestList.stream()
+ .filter(req -> req.getAllocationTags().contains(tag)).count();
+ if (count > max) {
+ max = count;
+ }
+ }
+ return max;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return (cursor < schedulingRequestList.size());
+ }
+
+ @Override
+ public SchedulingRequest next() {
+ if (hasNext()) {
+ return schedulingRequestList.get(cursor++);
+ }
+ throw new NoSuchElementException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/SerialIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/SerialIterator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/SerialIterator.java
new file mode 100644
index 0000000..68733a2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/SerialIterator.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.iterators;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+/**
+ * Traverse Scheduling Requests in the same order as they arrive
+ */
+public class SerialIterator implements Iterator<SchedulingRequest> {
+
+ private final List<SchedulingRequest> schedulingRequestList;
+ private int cursor;
+
+ public SerialIterator(Collection<SchedulingRequest> schedulingRequests) {
+ this.schedulingRequestList = new ArrayList<>(schedulingRequests);
+ this.cursor = 0;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return (cursor < schedulingRequestList.size());
+ }
+
+ @Override
+ public SchedulingRequest next() {
+ if (hasNext()) {
+ return schedulingRequestList.get(cursor++);
+ }
+ throw new NoSuchElementException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/package-info.java
new file mode 100644
index 0000000..c84671e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/iterators/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement
+ * contains classes related to scheduling containers using placement
+ * constraints.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.iterators;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/package-info.java
new file mode 100644
index 0000000..bb82077
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/algorithm/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement
+ * contains classes related to scheduling containers using placement
+ * constraints.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
index fe92d2f..8b04860 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/BatchedRequests.java
@@ -21,12 +21,15 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.iterators.PopularTagsIterator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.iterators.SerialIterator;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
import java.util.Set;
@@ -35,7 +38,8 @@ import java.util.Set;
* to place as a batch. The placement algorithm tends to give more optimal
* placements if more requests are batched together.
*/
-class BatchedRequests implements ConstraintPlacementAlgorithmInput {
+public class BatchedRequests
+ implements ConstraintPlacementAlgorithmInput, Iterable<SchedulingRequest> {
// PlacementAlgorithmOutput attempt - the number of times the requests in this
// batch has been placed but was rejected by the scheduler.
@@ -44,19 +48,46 @@ class BatchedRequests implements ConstraintPlacementAlgorithmInput {
private final ApplicationId applicationId;
private final Collection<SchedulingRequest> requests;
private final Map<String, Set<NodeId>> blacklist = new HashMap<>();
+ private IteratorType iteratorType;
- BatchedRequests(ApplicationId applicationId,
+ /**
+ * Iterator Type.
+ */
+ public enum IteratorType {
+ SERIAL,
+ POPULAR_TAGS
+ }
+
+ public BatchedRequests(IteratorType type, ApplicationId applicationId,
Collection<SchedulingRequest> requests, int attempt) {
+ this.iteratorType = type;
this.applicationId = applicationId;
this.requests = requests;
this.placementAttempt = attempt;
}
/**
+ * Exposes SchedulingRequest Iterator interface which can be used
+ * to traverse requests using different heuristics i.e. Tag Popularity
+ * @return SchedulingRequest Iterator.
+ */
+ @Override
+ public Iterator<SchedulingRequest> iterator() {
+ switch (this.iteratorType) {
+ case SERIAL:
+ return new SerialIterator(requests);
+ case POPULAR_TAGS:
+ return new PopularTagsIterator(requests);
+ default:
+ return null;
+ }
+ }
+
+ /**
* Get Application Id.
* @return Application Id.
*/
- ApplicationId getApplicationId() {
+ public ApplicationId getApplicationId() {
return applicationId;
}
@@ -73,11 +104,11 @@ class BatchedRequests implements ConstraintPlacementAlgorithmInput {
* Add a Scheduling request to the batch.
* @param req Scheduling Request.
*/
- void addToBatch(SchedulingRequest req) {
+ public void addToBatch(SchedulingRequest req) {
requests.add(req);
}
- void addToBlacklist(Set<String> tags, SchedulerNode node) {
+ public void addToBlacklist(Set<String> tags, SchedulerNode node) {
if (tags != null && !tags.isEmpty()) {
// We are currently assuming a single allocation tag
// per scheduler request currently.
@@ -90,7 +121,7 @@ class BatchedRequests implements ConstraintPlacementAlgorithmInput {
* Get placement attempt.
* @return PlacementAlgorithmOutput placement Attempt.
*/
- int getPlacementAttempt() {
+ public int getPlacementAttempt() {
return placementAttempt;
}
@@ -99,7 +130,7 @@ class BatchedRequests implements ConstraintPlacementAlgorithmInput {
* @param tag Tag.
* @return Set of blacklisted Nodes.
*/
- Set<NodeId> getBlacklist(String tag) {
+ public Set<NodeId> getBlacklist(String tag) {
return blacklist.getOrDefault(tag, Collections.EMPTY_SET);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
index d613d4e..8e9c79c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/PlacementProcessor.java
@@ -35,8 +35,10 @@ import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.algorithm.DefaultPlacementAlgorithm;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.SchedulingResponse;
@@ -98,6 +100,7 @@ public class PlacementProcessor implements ApplicationMasterServiceProcessor {
private Map<ApplicationId, List<SchedulingRequest>> requestsToReject =
new ConcurrentHashMap<>();
+ private BatchedRequests.IteratorType iteratorType;
private PlacementDispatcher placementDispatcher;
@@ -122,9 +125,20 @@ public class PlacementProcessor implements ApplicationMasterServiceProcessor {
if (instances != null && !instances.isEmpty()) {
algorithm = instances.get(0);
} else {
- algorithm = new SamplePlacementAlgorithm();
+ algorithm = new DefaultPlacementAlgorithm();
+ }
+ LOG.info("Placement Algorithm [{}]", algorithm.getClass().getName());
+
+ String iteratorName = ((RMContextImpl) amsContext).getYarnConfiguration()
+ .get(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ALGORITHM_ITERATOR,
+ BatchedRequests.IteratorType.SERIAL.name());
+ LOG.info("Placement Algorithm Iterator[{}]", iteratorName);
+ try {
+ iteratorType = BatchedRequests.IteratorType.valueOf(iteratorName);
+ } catch (IllegalArgumentException e) {
+ throw new YarnRuntimeException(
+ "Could not instantiate Placement Algorithm Iterator: ", e);
}
- LOG.info("Planning Algorithm [{}]", algorithm.getClass().getName());
int algoPSize = ((RMContextImpl) amsContext).getYarnConfiguration().getInt(
YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ALGORITHM_POOL_SIZE,
@@ -188,9 +202,8 @@ public class PlacementProcessor implements ApplicationMasterServiceProcessor {
private void dispatchRequestsForPlacement(ApplicationAttemptId appAttemptId,
List<SchedulingRequest> schedulingRequests) {
if (schedulingRequests != null && !schedulingRequests.isEmpty()) {
- this.placementDispatcher.dispatch(
- new BatchedRequests(appAttemptId.getApplicationId(),
- schedulingRequests, 1));
+ this.placementDispatcher.dispatch(new BatchedRequests(iteratorType,
+ appAttemptId.getApplicationId(), schedulingRequests, 1));
}
}
@@ -329,11 +342,10 @@ public class PlacementProcessor implements ApplicationMasterServiceProcessor {
}
}
if (!isAdded) {
- BatchedRequests br =
- new BatchedRequests(schedulerResponse.getApplicationId(),
- Collections.singleton(
- schedulerResponse.getSchedulingRequest()),
- placementAttempt + 1);
+ BatchedRequests br = new BatchedRequests(iteratorType,
+ schedulerResponse.getApplicationId(),
+ Collections.singleton(schedulerResponse.getSchedulingRequest()),
+ placementAttempt + 1);
reqsToRetry.add(br);
br.addToBlacklist(
schedulerResponse.getSchedulingRequest().getAllocationTags(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
deleted file mode 100644
index 8d49801..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/processor/SamplePlacementAlgorithm.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor;
-
-import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
-import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SpecializedConstraintTransformer;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithm;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmInput;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutput;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.ConstraintPlacementAlgorithmOutputCollector;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api.PlacedSchedulingRequest;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Sample Test algorithm. Assumes anti-affinity always
- * It also assumes the numAllocations in resource sizing is always = 1
- *
- * NOTE: This is just a sample implementation. Not be actually used
- */
-public class SamplePlacementAlgorithm implements ConstraintPlacementAlgorithm {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SamplePlacementAlgorithm.class);
-
- private AllocationTagsManager tagsManager;
- private PlacementConstraintManager constraintManager;
- private NodeCandidateSelector nodeSelector;
-
- @Override
- public void init(RMContext rmContext) {
- this.tagsManager = rmContext.getAllocationTagsManager();
- this.constraintManager = rmContext.getPlacementConstraintManager();
- this.nodeSelector =
- filter -> ((AbstractYarnScheduler)(rmContext)
- .getScheduler()).getNodes(filter);
- }
-
- @Override
- public void place(ConstraintPlacementAlgorithmInput input,
- ConstraintPlacementAlgorithmOutputCollector collector) {
- BatchedRequests requests = (BatchedRequests)input;
- ConstraintPlacementAlgorithmOutput resp =
- new ConstraintPlacementAlgorithmOutput(requests.getApplicationId());
- List<SchedulerNode> allNodes = nodeSelector.selectNodes(null);
- Map<String, List<SchedulingRequest>> tagIndexedRequests = new HashMap<>();
- requests.getSchedulingRequests()
- .stream()
- .filter(r -> r.getAllocationTags() != null)
- .forEach(
- req -> req.getAllocationTags().forEach(
- tag -> tagIndexedRequests.computeIfAbsent(tag,
- k -> new ArrayList<>()).add(req))
- );
- for (Map.Entry<String, List<SchedulingRequest>> entry :
- tagIndexedRequests.entrySet()) {
- String tag = entry.getKey();
- PlacementConstraint constraint =
- constraintManager.getConstraint(requests.getApplicationId(),
- Collections.singleton(tag));
- if (constraint != null) {
- // Currently works only for simple anti-affinity
- // NODE scope target expressions
- SpecializedConstraintTransformer transformer =
- new SpecializedConstraintTransformer(constraint);
- PlacementConstraint transform = transformer.transform();
- TargetConstraint targetConstraint =
- (TargetConstraint) transform.getConstraintExpr();
- // Assume a single target expression tag;
- // The Sample Algorithm assumes a constraint will always be a simple
- // Target Constraint with a single entry in the target set.
- // As mentioned in the class javadoc - This algorithm should be
- // used mostly for testing and validating end-2-end workflow.
- String targetTag =
- targetConstraint.getTargetExpressions().iterator().next()
- .getTargetValues().iterator().next();
- // iterate over all nodes
- Iterator<SchedulerNode> nodeIter = allNodes.iterator();
- List<SchedulingRequest> schedulingRequests = entry.getValue();
- Iterator<SchedulingRequest> reqIter = schedulingRequests.iterator();
- while (reqIter.hasNext()) {
- SchedulingRequest sReq = reqIter.next();
- int numAllocs = sReq.getResourceSizing().getNumAllocations();
- while (numAllocs > 0 && nodeIter.hasNext()) {
- SchedulerNode node = nodeIter.next();
- long nodeCardinality = 0;
- try {
- nodeCardinality = tagsManager.getNodeCardinality(
- node.getNodeID(), requests.getApplicationId(),
- targetTag);
- if (nodeCardinality == 0 &&
- !requests.getBlacklist(tag).contains(node.getNodeID())) {
- numAllocs--;
- sReq.getResourceSizing().setNumAllocations(numAllocs);
- PlacedSchedulingRequest placedReq =
- new PlacedSchedulingRequest(sReq);
- placedReq.setPlacementAttempt(requests.getPlacementAttempt());
- placedReq.getNodes().add(node);
- resp.getPlacedRequests().add(placedReq);
- }
- } catch (InvalidAllocationTagsQueryException e) {
- LOG.warn("Got exception from TagManager !", e);
- }
- }
- }
- }
- }
- // Add all requests whose numAllocations still > 0 to rejected list.
- requests.getSchedulingRequests().stream()
- .filter(sReq -> sReq.getResourceSizing().getNumAllocations() > 0)
- .forEach(rejReq -> resp.getRejectedRequests().add(rejReq));
- collector.collect(resp);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
index 0ce1614..f1d5663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
@@ -75,24 +75,24 @@ public class TestAllocationTagsManager {
// 3 Containers from app1
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
// 1 Container from app2
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
// Get Node Cardinality of app1 on node1, with tag "mapper"
@@ -170,24 +170,21 @@ public class TestAllocationTagsManager {
// Finish all containers:
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Expect all cardinality to be 0
// Get Cardinality of app1 on node1, with tag "mapper"
@@ -270,25 +267,22 @@ public class TestAllocationTagsManager {
// 3 Containers from app1
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 2),
+ TestUtils.getMockContainerId(2, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(2, 4), ImmutableSet.of("reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
// 1 Container from app2
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Get Rack Cardinality of app1 on rack0, with tag "mapper"
Assert.assertEquals(1, atm.getRackCardinality("rack0",
@@ -325,45 +319,39 @@ public class TestAllocationTagsManager {
// Add a bunch of containers
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Remove all these containers
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.removeContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.removeContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// Check internal data structure
Assert.assertEquals(0,
@@ -375,6 +363,87 @@ public class TestAllocationTagsManager {
}
@Test
+ public void testTempContainerAllocations()
+ throws InvalidAllocationTagsQueryException {
+ /**
+ * Construct both TEMP and normal containers: Node1: TEMP container_1_1
+ * (mapper/reducer/app_1) container_1_2 (service/app_1)
+ *
+ * Node2: container_1_3 (reducer/app_1) TEMP container_2_1 (service/app_2)
+ */
+
+ AllocationTagsManager atm = new AllocationTagsManager(rmContext);
+
+ // 3 Containers from app1
+ atm.addTempContainer(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("host1:123"),
+ TestUtils.getMockContainerId(1, 2), ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("reducer"));
+
+ // 1 Container from app2
+ atm.addTempContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"));
+
+ // Expect tag mappings to be present including temp Tags
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::sum));
+
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("service"),
+ Long::sum));
+
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"),
+ Long::sum));
+
+ // Do a temp Tag cleanup on app2
+ atm.cleanTempContainers(TestUtils.getMockApplicationId(2));
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"),
+ Long::sum));
+ // Expect app1 to be unaffected
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::sum));
+ // Do a cleanup on app1 as well
+ atm.cleanTempContainers(TestUtils.getMockApplicationId(1));
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::sum));
+
+ // Non temp-tags should be unaffected
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("service"),
+ Long::sum));
+
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("service"),
+ Long::sum));
+
+ // Expect app2 with no containers, and app1 with 2 containers across 2 nodes
+ Assert.assertEquals(2,
+ atm.getPerAppNodeMappings().get(TestUtils.getMockApplicationId(1))
+ .getTypeToTagsWithCount().size());
+
+ Assert.assertNull(
+ atm.getPerAppNodeMappings().get(TestUtils.getMockApplicationId(2)));
+ }
+
+ @Test
public void testQueryCardinalityWithIllegalParameters()
throws InvalidAllocationTagsQueryException {
/**
@@ -385,24 +454,21 @@ public class TestAllocationTagsManager {
// Add a bunch of containers
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
atm.addContainer(NodeId.fromString("host1:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(1, 3), ImmutableSet.of("service"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
+ TestUtils.getMockContainerId(1, 4), ImmutableSet.of("reducer"));
atm.addContainer(NodeId.fromString("host2:123"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
+ TestUtils.getMockContainerId(2, 3), ImmutableSet.of("service"));
// No node-id
boolean caughtException = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java
new file mode 100644
index 0000000..0e7b715
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestBatchedRequestsIterators.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.TestPlacementProcessor.schedulingRequest;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.processor.BatchedRequests;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test Request Iterator.
+ */
+public class TestBatchedRequestsIterators {
+
+ @Test
+ public void testSerialIterator() throws Exception {
+ List<SchedulingRequest> schedulingRequestList =
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 4, 1, 512, "foo"));
+
+ BatchedRequests batchedRequests = new BatchedRequests(
+ BatchedRequests.IteratorType.SERIAL, null, schedulingRequestList, 1);
+
+ Iterator<SchedulingRequest> requestIterator = batchedRequests.iterator();
+ long prevAllocId = 0;
+ while (requestIterator.hasNext()) {
+ SchedulingRequest request = requestIterator.next();
+ Assert.assertTrue(request.getAllocationRequestId() > prevAllocId);
+ prevAllocId = request.getAllocationRequestId();
+ }
+ }
+
+ @Test
+ public void testPopularTagsIterator() throws Exception {
+ List<SchedulingRequest> schedulingRequestList =
+ Arrays.asList(schedulingRequest(1, 1, 1, 512, "pri", "foo"),
+ schedulingRequest(1, 2, 1, 512, "bar"),
+ schedulingRequest(1, 3, 1, 512, "foo", "pri"),
+ schedulingRequest(1, 4, 1, 512, "test"),
+ schedulingRequest(1, 5, 1, 512, "pri", "bar"));
+
+ BatchedRequests batchedRequests =
+ new BatchedRequests(BatchedRequests.IteratorType.POPULAR_TAGS, null,
+ schedulingRequestList, 1);
+
+ Iterator<SchedulingRequest> requestIterator = batchedRequests.iterator();
+ long recCcount = 0;
+ while (requestIterator.hasNext()) {
+ SchedulingRequest request = requestIterator.next();
+ if (recCcount < 3) {
+ Assert.assertTrue(request.getAllocationTags().contains("pri"));
+ } else {
+ Assert.assertTrue(request.getAllocationTags().contains("bar")
+ || request.getAllocationTags().contains("test"));
+ }
+ recCcount++;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bcd5cc1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
index db8ae15..87dd5b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementProcessor.java
@@ -373,13 +373,13 @@ public class TestPlacementProcessor {
rej.getReason());
}
- private static SchedulingRequest schedulingRequest(
+ protected static SchedulingRequest schedulingRequest(
int priority, long allocReqId, int cores, int mem, String... tags) {
return schedulingRequest(priority, allocReqId, cores, mem,
ExecutionType.GUARANTEED, tags);
}
- private static SchedulingRequest schedulingRequest(
+ protected static SchedulingRequest schedulingRequest(
int priority, long allocReqId, int cores, int mem,
ExecutionType execType, String... tags) {
return SchedulingRequest.newBuilder()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/11] hadoop git commit: YARN-7669. API and interface modifications
for placement constraint processor. (asuresh)
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
new file mode 100644
index 0000000..9571f0e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutput.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Encapsulates the output of the ConstraintPlacementAlgorithm. The Algorithm
+ * is free to produce multiple of output objects at the end of each run and it
+ * must use the provided ConstraintPlacementAlgorithmOutputCollector to
+ * aggregate/collect this output. Similar to the MapReduce Mapper/Reducer
+ * which is provided a collector to collect output.
+ */
+public class ConstraintPlacementAlgorithmOutput {
+
+ private final ApplicationId applicationId;
+
+ public ConstraintPlacementAlgorithmOutput(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ private final List<PlacedSchedulingRequest> placedRequests =
+ new ArrayList<>();
+
+ private final List<SchedulingRequest> rejectedRequests =
+ new ArrayList<>();
+
+ public List<PlacedSchedulingRequest> getPlacedRequests() {
+ return placedRequests;
+ }
+
+ public List<SchedulingRequest> getRejectedRequests() {
+ return rejectedRequests;
+ }
+
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
new file mode 100644
index 0000000..131fd42
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmOutputCollector.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+/**
+ * The ConstraintPlacementAlgorithm uses the
+ * ConstraintPlacementAlgorithmOutputCollector to collect any output it
+ * spits out.
+ */
+public interface ConstraintPlacementAlgorithmOutputCollector {
+
+ /**
+ * Collect an ConstraintPlacementAlgorithm output.
+ * @param algorithmOutput ConstraintPlacementAlgorithm Output.
+ */
+ void collect(ConstraintPlacementAlgorithmOutput algorithmOutput);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java
new file mode 100644
index 0000000..2cd90d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/PlacedSchedulingRequest.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Class to encapsulate a Placed scheduling Request.
+ * It has the original Scheduling Request and a list of SchedulerNodes (one
+ * for each 'numAllocation' field in the corresponding ResourceSizing object.
+ *
+ * NOTE: Clients of this class SHOULD NOT rely on the value of
+ * resourceSizing.numAllocations and instead should use the
+ * size of collection returned by getNodes() instead.
+ */
+public class PlacedSchedulingRequest {
+
+ // The number of times the Algorithm tried to place the SchedulingRequest
+ // after it was rejected by the commit phase of the Scheduler (due to some
+ // transient state of the cluster. For eg: no space on Node / user limit etc.)
+ // The Algorithm can then try to probably place on a different node.
+ private int placementAttempt = 0;
+ private final SchedulingRequest request;
+ // One Node per numContainers in the SchedulingRequest;
+ private final List<SchedulerNode> nodes = new ArrayList<>();
+
+ public PlacedSchedulingRequest(SchedulingRequest request) {
+ this.request = request;
+ }
+
+ public SchedulingRequest getSchedulingRequest() {
+ return request;
+ }
+
+ /**
+ * List of Node locations on which this Scheduling Request can be placed.
+ * The size of this list = schedulingRequest.resourceSizing.numAllocations.
+ * @return List of Scheduler nodes.
+ */
+ public List<SchedulerNode> getNodes() {
+ return nodes;
+ }
+
+ public int getPlacementAttempt() {
+ return placementAttempt;
+ }
+
+ public void setPlacementAttempt(int attempt) {
+ this.placementAttempt = attempt;
+ }
+
+ @Override
+ public String toString() {
+ return "PlacedSchedulingRequest{" +
+ "placementAttempt=" + placementAttempt +
+ ", request=" + request +
+ ", nodes=" + nodes +
+ '}';
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java
new file mode 100644
index 0000000..6c65d84
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/SchedulingResponse.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+/**
+ * This class encapsulates the response received from the ResourceScheduler's
+ * attemptAllocateOnNode method.
+ */
+public class SchedulingResponse {
+
+ private final boolean isSuccess;
+ private final ApplicationId applicationId;
+ private final SchedulingRequest schedulingRequest;
+
+ /**
+ * Create a SchedulingResponse.
+ * @param isSuccess did scheduler accept.
+ * @param applicationId Application Id.
+ * @param schedulingRequest Scheduling Request.
+ */
+ public SchedulingResponse(boolean isSuccess, ApplicationId applicationId,
+ SchedulingRequest schedulingRequest) {
+ this.isSuccess = isSuccess;
+ this.applicationId = applicationId;
+ this.schedulingRequest = schedulingRequest;
+ }
+
+ /**
+ * Returns true if Scheduler was able to accept and commit this request.
+ * @return isSuccessful.
+ */
+ public boolean isSuccess() {
+ return this.isSuccess;
+ }
+
+ /**
+ * Get Application Id.
+ * @return Application Id.
+ */
+ public ApplicationId getApplicationId() {
+ return this.applicationId;
+ }
+
+ /**
+ * Get Scheduling Request.
+ * @return Scheduling Request.
+ */
+ public SchedulingRequest getSchedulingRequest() {
+ return this.schedulingRequest;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java
new file mode 100644
index 0000000..01ed713
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This Package contains classes related to constrained placement of
+ * Requests.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java
deleted file mode 100644
index 0358792..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/TestAllocationTagsManager.java
+++ /dev/null
@@ -1,328 +0,0 @@
-/*
- * *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * /
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.constraint;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- * Test functionality of AllocationTagsManager.
- */
-public class TestAllocationTagsManager {
- @Test
- public void testAllocationTagsManagerSimpleCases()
- throws InvalidAllocationTagsQueryException {
- AllocationTagsManager atm = new AllocationTagsManager();
-
- /**
- * Construct test case:
- * Node1:
- * container_1_1 (mapper/reducer/app_1)
- * container_1_3 (service/app_1)
- *
- * Node2:
- * container_1_2 (mapper/reducer/app_1)
- * container_1_4 (reducer/app_1)
- * container_2_1 (service/app_2)
- */
-
- // 3 Containers from app1
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- // 1 Container from app2
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Get Cardinality of app1 on node1, with tag "mapper"
- Assert.assertEquals(1,
- atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
- Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
- Assert.assertEquals(1,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::min));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
- Assert.assertEquals(3,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::sum));
-
- // Get Cardinality by passing single tag.
- Assert.assertEquals(1,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), "mapper"));
-
- Assert.assertEquals(2,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), "reducer"));
-
- // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("no_existed", "reducer"), Long::min));
-
- // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
- // (Expect this returns #containers from app1 on node2)
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet
- .of(AllocationTagsNamespaces.APP_ID + TestUtils
- .getMockApplicationId(1).toString()), Long::max));
-
- // Get Cardinality of app1 on node2, with empty tag set, op=max
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
-
- // Get Cardinality of all apps on node2, with empty tag set, op=sum
- Assert.assertEquals(7,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
- ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(5,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
-
- // Finish all containers:
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Expect all cardinality to be 0
- // Get Cardinality of app1 on node1, with tag "mapper"
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
- Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::min));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::max));
-
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of("mapper", "reducer"), Long::sum));
-
- // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
- // (Expect this returns #containers from app1 on node2)
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()),
- Long::max));
-
- Assert.assertEquals(0,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1),
- TestUtils.getMockApplicationId(1).toString()));
-
- // Get Cardinality of app1 on node2, with empty tag set, op=max
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
-
- // Get Cardinality of all apps on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
- ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
-
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
- }
-
- @Test
- public void testAllocationTagsManagerMemoryAfterCleanup()
- throws InvalidAllocationTagsQueryException {
- /**
- * Make sure YARN cleans up all memory once container/app finishes.
- */
-
- AllocationTagsManager atm = new AllocationTagsManager();
-
- // Add a bunch of containers
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Remove all these containers
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.removeContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.removeContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // Check internal data structure
- Assert.assertEquals(0,
- atm.getGlobalMapping().getNodeToTagsWithCount().size());
- Assert.assertEquals(0, atm.getPerAppMappings().size());
- }
-
- @Test
- public void testQueryCardinalityWithIllegalParameters()
- throws InvalidAllocationTagsQueryException {
- /**
- * Make sure YARN cleans up all memory once container/app finishes.
- */
-
- AllocationTagsManager atm = new AllocationTagsManager();
-
- // Add a bunch of containers
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
- ImmutableSet.of("mapper", "reducer"));
-
- atm.addContainer(NodeId.fromString("node1:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
- ImmutableSet.of("service"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
- ImmutableSet.of("reducer"));
-
- atm.addContainer(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
- ImmutableSet.of("service"));
-
- // No node-id
- boolean caughtException = false;
- try {
- atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2),
- ImmutableSet.of("mapper"), Long::min);
- } catch (InvalidAllocationTagsQueryException e) {
- caughtException = true;
- }
- Assert.assertTrue("should fail because of nodeId specified",
- caughtException);
-
- // No op
- caughtException = false;
- try {
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null);
- } catch (InvalidAllocationTagsQueryException e) {
- caughtException = true;
- }
- Assert.assertTrue("should fail because of nodeId specified",
- caughtException);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 27ff311..538d128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -62,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 61a5555..e8734cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -42,12 +42,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
new file mode 100644
index 0000000..4bb2a18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
@@ -0,0 +1,328 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test functionality of AllocationTagsManager.
+ */
+public class TestAllocationTagsManager {
+ @Test
+ public void testAllocationTagsManagerSimpleCases()
+ throws InvalidAllocationTagsQueryException {
+ AllocationTagsManager atm = new AllocationTagsManager();
+
+ /**
+ * Construct test case:
+ * Node1:
+ * container_1_1 (mapper/reducer/app_1)
+ * container_1_3 (service/app_1)
+ *
+ * Node2:
+ * container_1_2 (mapper/reducer/app_1)
+ * container_1_4 (reducer/app_1)
+ * container_2_1 (service/app_2)
+ */
+
+ // 3 Containers from app1
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ // 1 Container from app2
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Get Cardinality of app1 on node1, with tag "mapper"
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
+ Assert.assertEquals(1,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::min));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
+ Assert.assertEquals(3,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::sum));
+
+ // Get Cardinality by passing single tag.
+ Assert.assertEquals(1,
+ atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), "mapper"));
+
+ Assert.assertEquals(2,
+ atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), "reducer"));
+
+ // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("no_existed", "reducer"), Long::min));
+
+ // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
+ // (Expect this returns #containers from app1 on node2)
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet
+ .of(AllocationTagsNamespaces.APP_ID + TestUtils
+ .getMockApplicationId(1).toString()), Long::max));
+
+ // Get Cardinality of app1 on node2, with empty tag set, op=max
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
+
+ // Get Cardinality of all apps on node2, with empty tag set, op=sum
+ Assert.assertEquals(7,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
+ ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(5,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
+
+ // Finish all containers:
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Expect all cardinality to be 0
+ // Get Cardinality of app1 on node1, with tag "mapper"
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
+ Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::min));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::max));
+
+ // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of("mapper", "reducer"), Long::sum));
+
+ // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
+ // (Expect this returns #containers from app1 on node2)
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()),
+ Long::max));
+
+ Assert.assertEquals(0,
+ atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1),
+ TestUtils.getMockApplicationId(1).toString()));
+
+ // Get Cardinality of app1 on node2, with empty tag set, op=max
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
+
+ // Get Cardinality of all apps on node2, with empty tag set, op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
+ ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
+
+ // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ Assert.assertEquals(0,
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
+ }
+
+ @Test
+ public void testAllocationTagsManagerMemoryAfterCleanup()
+ throws InvalidAllocationTagsQueryException {
+ /**
+ * Make sure YARN cleans up all memory once container/app finishes.
+ */
+
+ AllocationTagsManager atm = new AllocationTagsManager();
+
+ // Add a bunch of containers
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Remove all these containers
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.removeContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.removeContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Check internal data structure
+ Assert.assertEquals(0,
+ atm.getGlobalMapping().getNodeToTagsWithCount().size());
+ Assert.assertEquals(0, atm.getPerAppMappings().size());
+ }
+
+ @Test
+ public void testQueryCardinalityWithIllegalParameters()
+ throws InvalidAllocationTagsQueryException {
+ /**
+ * Make sure YARN cleans up all memory once container/app finishes.
+ */
+
+ AllocationTagsManager atm = new AllocationTagsManager();
+
+ // Add a bunch of containers
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("node1:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.addContainer(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // No node-id
+ boolean caughtException = false;
+ try {
+ atm.getNodeCardinalityByOp(null, TestUtils.getMockApplicationId(2),
+ ImmutableSet.of("mapper"), Long::min);
+ } catch (InvalidAllocationTagsQueryException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue("should fail because of nodeId specified",
+ caughtException);
+
+ // No op
+ caughtException = false;
+ try {
+ atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null);
+ } catch (InvalidAllocationTagsQueryException e) {
+ caughtException = true;
+ }
+ Assert.assertTrue("should fail because of nodeId specified",
+ caughtException);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 4b902a7..db749ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -74,7 +74,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -93,6 +92,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/11] hadoop git commit: YARN-7653. Node group support for
AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
Posted by as...@apache.org.
YARN-7653. Node group support for AllocationTagsManager. (Panagiotis Garefalakis via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3dd5204
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3dd5204
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3dd5204
Branch: refs/heads/YARN-6592
Commit: b3dd5204baedd75b831def755e9b0cd01a3a3ac4
Parents: d70b046
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 22 07:24:37 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:50:16 2018 -0800
----------------------------------------------------------------------
.../server/resourcemanager/ResourceManager.java | 2 +-
.../constraint/AllocationTagsManager.java | 282 ++++++++++++++-----
.../rmcontainer/TestRMContainerImpl.java | 2 +-
.../constraint/TestAllocationTagsManager.java | 269 ++++++++++++------
4 files changed, 392 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3dd5204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index adda465..d71f224 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -493,7 +493,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected AllocationTagsManager createAllocationTagsManager() {
- return new AllocationTagsManager();
+ return new AllocationTagsManager(this.rmContext);
}
protected DelegationTokenRenewer createDelegationTokenRenewer() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3dd5204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
index c278606..7b0b959 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.log4j.Logger;
import java.util.HashMap;
@@ -38,9 +39,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.LongBinaryOperator;
/**
- * Support storing maps between container-tags/applications and
- * nodes. This will be required by affinity/anti-affinity implementation and
- * cardinality.
+ * In-memory mapping between applications/container-tags and nodes/racks.
+ * Required by constrained affinity/anti-affinity and cardinality placement.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -51,48 +51,54 @@ public class AllocationTagsManager {
private ReentrantReadWriteLock.ReadLock readLock;
private ReentrantReadWriteLock.WriteLock writeLock;
+ private final RMContext rmContext;
- // Application's tags to node
- private Map<ApplicationId, NodeToCountedTags> perAppMappings =
+ // Application's tags to Node
+ private Map<ApplicationId, NodeToCountedTags> perAppNodeMappings =
+ new HashMap<>();
+ // Application's tags to Rack
+ private Map<ApplicationId, NodeToCountedTags> perAppRackMappings =
new HashMap<>();
// Global tags to node mapping (used to fast return aggregated tags
// cardinality across apps)
- private NodeToCountedTags globalMapping = new NodeToCountedTags();
+ private NodeToCountedTags<NodeId> globalNodeMapping = new NodeToCountedTags();
+ // Global tags to Rack mapping
+ private NodeToCountedTags<String> globalRackMapping = new NodeToCountedTags();
/**
- * Store node to counted tags.
+ * Generic store mapping type <T> to counted tags.
+ * Currently used both for NodeId to Tag, Count and Rack to Tag, Count
*/
@VisibleForTesting
- static class NodeToCountedTags {
- // Map<NodeId, Map<Tag, Count>>
- private Map<NodeId, Map<String, Long>> nodeToTagsWithCount =
- new HashMap<>();
+ static class NodeToCountedTags<T> {
+ // Map<Type, Map<Tag, Count>>
+ private Map<T, Map<String, Long>> typeToTagsWithCount = new HashMap<>();
// protected by external locks
- private void addTagsToNode(NodeId nodeId, Set<String> tags) {
- Map<String, Long> innerMap = nodeToTagsWithCount.computeIfAbsent(nodeId,
- k -> new HashMap<>());
+ private void addTags(T type, Set<String> tags) {
+ Map<String, Long> innerMap =
+ typeToTagsWithCount.computeIfAbsent(type, k -> new HashMap<>());
for (String tag : tags) {
Long count = innerMap.get(tag);
if (count == null) {
innerMap.put(tag, 1L);
- } else{
+ } else {
innerMap.put(tag, count + 1);
}
}
}
// protected by external locks
- private void addTagToNode(NodeId nodeId, String tag) {
- Map<String, Long> innerMap = nodeToTagsWithCount.computeIfAbsent(nodeId,
- k -> new HashMap<>());
+ private void addTag(T type, String tag) {
+ Map<String, Long> innerMap =
+ typeToTagsWithCount.computeIfAbsent(type, k -> new HashMap<>());
Long count = innerMap.get(tag);
if (count == null) {
innerMap.put(tag, 1L);
- } else{
+ } else {
innerMap.put(tag, count + 1);
}
}
@@ -104,17 +110,17 @@ public class AllocationTagsManager {
} else {
if (count <= 0) {
LOG.warn(
- "Trying to remove tags from node, however the count already"
+ "Trying to remove tags from node/rack, however the count already"
+ " becomes 0 or less, it could be a potential bug.");
}
innerMap.remove(tag);
}
}
- private void removeTagsFromNode(NodeId nodeId, Set<String> tags) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ private void removeTags(T type, Set<String> tags) {
+ Map<String, Long> innerMap = typeToTagsWithCount.get(type);
if (innerMap == null) {
- LOG.warn("Failed to find node=" + nodeId
+ LOG.warn("Failed to find node/rack=" + type
+ " while trying to remove tags, please double check.");
return;
}
@@ -124,14 +130,14 @@ public class AllocationTagsManager {
}
if (innerMap.isEmpty()) {
- nodeToTagsWithCount.remove(nodeId);
+ typeToTagsWithCount.remove(type);
}
}
- private void removeTagFromNode(NodeId nodeId, String tag) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ private void removeTag(T type, String tag) {
+ Map<String, Long> innerMap = typeToTagsWithCount.get(type);
if (innerMap == null) {
- LOG.warn("Failed to find node=" + nodeId
+ LOG.warn("Failed to find node/rack=" + type
+ " while trying to remove tags, please double check.");
return;
}
@@ -139,12 +145,12 @@ public class AllocationTagsManager {
removeTagFromInnerMap(innerMap, tag);
if (innerMap.isEmpty()) {
- nodeToTagsWithCount.remove(nodeId);
+ typeToTagsWithCount.remove(type);
}
}
- private long getCardinality(NodeId nodeId, String tag) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ private long getCardinality(T type, String tag) {
+ Map<String, Long> innerMap = typeToTagsWithCount.get(type);
if (innerMap == null) {
return 0;
}
@@ -152,9 +158,9 @@ public class AllocationTagsManager {
return value == null ? 0 : value;
}
- private long getCardinality(NodeId nodeId, Set<String> tags,
+ private long getCardinality(T type, Set<String> tags,
LongBinaryOperator op) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ Map<String, Long> innerMap = typeToTagsWithCount.get(type);
if (innerMap == null) {
return 0;
}
@@ -193,29 +199,40 @@ public class AllocationTagsManager {
}
private boolean isEmpty() {
- return nodeToTagsWithCount.isEmpty();
+ return typeToTagsWithCount.isEmpty();
}
@VisibleForTesting
- public Map<NodeId, Map<String, Long>> getNodeToTagsWithCount() {
- return nodeToTagsWithCount;
+ public Map<T, Map<String, Long>> getTypeToTagsWithCount() {
+ return typeToTagsWithCount;
}
}
@VisibleForTesting
- Map<ApplicationId, NodeToCountedTags> getPerAppMappings() {
- return perAppMappings;
+ Map<ApplicationId, NodeToCountedTags> getPerAppNodeMappings() {
+ return perAppNodeMappings;
+ }
+
+ @VisibleForTesting
+ Map<ApplicationId, NodeToCountedTags> getPerAppRackMappings() {
+ return perAppRackMappings;
+ }
+
+ @VisibleForTesting
+ NodeToCountedTags getGlobalNodeMapping() {
+ return globalNodeMapping;
}
@VisibleForTesting
- NodeToCountedTags getGlobalMapping() {
- return globalMapping;
+ NodeToCountedTags getGlobalRackMapping() {
+ return globalRackMapping;
}
- public AllocationTagsManager() {
+ public AllocationTagsManager(RMContext context) {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
+ rmContext = context;
}
/**
@@ -243,21 +260,30 @@ public class AllocationTagsManager {
writeLock.lock();
try {
- NodeToCountedTags perAppTagsMapping = perAppMappings.computeIfAbsent(
- applicationId, k -> new NodeToCountedTags());
-
+ NodeToCountedTags perAppTagsMapping = perAppNodeMappings
+ .computeIfAbsent(applicationId, k -> new NodeToCountedTags());
+ NodeToCountedTags perAppRackTagsMapping = perAppRackMappings
+ .computeIfAbsent(applicationId, k -> new NodeToCountedTags());
+ // Covering test-cases where context is mocked
+ String nodeRack = (rmContext.getRMNodes() != null
+ && rmContext.getRMNodes().get(nodeId) != null)
+ ? rmContext.getRMNodes().get(nodeId).getRackName()
+ : "default-rack";
if (useSet) {
- perAppTagsMapping.addTagsToNode(nodeId, allocationTags);
- globalMapping.addTagsToNode(nodeId, allocationTags);
+ perAppTagsMapping.addTags(nodeId, allocationTags);
+ perAppRackTagsMapping.addTags(nodeRack, allocationTags);
+ globalNodeMapping.addTags(nodeId, allocationTags);
+ globalRackMapping.addTags(nodeRack, allocationTags);
} else {
- perAppTagsMapping.addTagToNode(nodeId, applicationIdTag);
- globalMapping.addTagToNode(nodeId, applicationIdTag);
+ perAppTagsMapping.addTag(nodeId, applicationIdTag);
+ perAppRackTagsMapping.addTag(nodeRack, applicationIdTag);
+ globalNodeMapping.addTag(nodeId, applicationIdTag);
+ globalRackMapping.addTag(nodeRack, applicationIdTag);
}
if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Added container=" + containerId + " with tags=[" + StringUtils
- .join(allocationTags, ",") + "]");
+ LOG.debug("Added container=" + containerId + " with tags=["
+ + StringUtils.join(allocationTags, ",") + "]");
}
} finally {
writeLock.unlock();
@@ -287,27 +313,40 @@ public class AllocationTagsManager {
writeLock.lock();
try {
- NodeToCountedTags perAppTagsMapping = perAppMappings.get(applicationId);
+ NodeToCountedTags perAppTagsMapping =
+ perAppNodeMappings.get(applicationId);
+ NodeToCountedTags perAppRackTagsMapping =
+ perAppRackMappings.get(applicationId);
if (perAppTagsMapping == null) {
return;
}
-
+ // Covering test-cases where context is mocked
+ String nodeRack = (rmContext.getRMNodes() != null
+ && rmContext.getRMNodes().get(nodeId) != null)
+ ? rmContext.getRMNodes().get(nodeId).getRackName()
+ : "default-rack";
if (useSet) {
- perAppTagsMapping.removeTagsFromNode(nodeId, allocationTags);
- globalMapping.removeTagsFromNode(nodeId, allocationTags);
+ perAppTagsMapping.removeTags(nodeId, allocationTags);
+ perAppRackTagsMapping.removeTags(nodeRack, allocationTags);
+ globalNodeMapping.removeTags(nodeId, allocationTags);
+ globalRackMapping.removeTags(nodeRack, allocationTags);
} else {
- perAppTagsMapping.removeTagFromNode(nodeId, applicationIdTag);
- globalMapping.removeTagFromNode(nodeId, applicationIdTag);
+ perAppTagsMapping.removeTag(nodeId, applicationIdTag);
+ perAppRackTagsMapping.removeTag(nodeRack, applicationIdTag);
+ globalNodeMapping.removeTag(nodeId, applicationIdTag);
+ globalRackMapping.removeTag(nodeRack, applicationIdTag);
}
if (perAppTagsMapping.isEmpty()) {
- perAppMappings.remove(applicationId);
+ perAppNodeMappings.remove(applicationId);
+ }
+ if (perAppRackTagsMapping.isEmpty()) {
+ perAppRackMappings.remove(applicationId);
}
if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Removed container=" + containerId + " with tags=[" + StringUtils
- .join(allocationTags, ",") + "]");
+ LOG.debug("Removed container=" + containerId + " with tags=["
+ + StringUtils.join(allocationTags, ",") + "]");
}
} finally {
writeLock.unlock();
@@ -315,18 +354,16 @@ public class AllocationTagsManager {
}
/**
- * Get cardinality for following conditions. External can pass-in a binary op
- * to implement customized logic. *
+ * Get Node cardinality for a specific tag.
+ * When applicationId is null, method returns aggregated cardinality
+ *
* @param nodeId nodeId, required.
* @param applicationId applicationId. When null is specified, return
* aggregated cardinality among all nodes.
* @param tag allocation tag, see
* {@link SchedulingRequest#getAllocationTags()},
- * When multiple tags specified. Returns cardinality
- * depends on op. If a specified tag doesn't exist,
- * 0 will be its cardinality.
- * When null/empty tags specified, all tags
- * (of the node/app) will be considered.
+ * If a specified tag doesn't exist,
+ * method returns 0.
* @return cardinality of specified query on the node.
* @throws InvalidAllocationTagsQueryException when illegal query
* parameter specified
@@ -338,14 +375,14 @@ public class AllocationTagsManager {
try {
if (nodeId == null) {
throw new InvalidAllocationTagsQueryException(
- "Must specify nodeId/tags/op to query cardinality");
+ "Must specify nodeId/tag to query cardinality");
}
NodeToCountedTags mapping;
if (applicationId != null) {
- mapping = perAppMappings.get(applicationId);
- } else{
- mapping = globalMapping;
+ mapping = perAppNodeMappings.get(applicationId);
+ } else {
+ mapping = globalNodeMapping;
}
if (mapping == null) {
@@ -359,11 +396,54 @@ public class AllocationTagsManager {
}
/**
+ * Get Rack cardinality for a specific tag.
+ *
+ * @param rack rack, required.
+ * @param applicationId applicationId. When null is specified, return
+ * aggregated cardinality among all nodes.
+ * @param tag allocation tag, see
+ * {@link SchedulingRequest#getAllocationTags()},
+ * If a specified tag doesn't exist,
+ * method returns 0.
+ * @return cardinality of specified query on the rack.
+ * @throws InvalidAllocationTagsQueryException when illegal query
+ * parameter specified
+ */
+ public long getRackCardinality(String rack, ApplicationId applicationId,
+ String tag) throws InvalidAllocationTagsQueryException {
+ readLock.lock();
+
+ try {
+ if (rack == null) {
+ throw new InvalidAllocationTagsQueryException(
+ "Must specify rack/tag to query cardinality");
+ }
+
+ NodeToCountedTags mapping;
+ if (applicationId != null) {
+ mapping = perAppRackMappings.get(applicationId);
+ } else {
+ mapping = globalRackMapping;
+ }
+
+ if (mapping == null) {
+ return 0;
+ }
+
+ return mapping.getCardinality(rack, tag);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+
+
+ /**
* Check if given tag exists on node.
*
* @param nodeId nodeId, required.
* @param applicationId applicationId. When null is specified, return
- * aggregated cardinality among all nodes.
+ * aggregation among all applications.
* @param tag allocation tag, see
* {@link SchedulingRequest#getAllocationTags()},
* When multiple tags specified. Returns cardinality
@@ -387,7 +467,7 @@ public class AllocationTagsManager {
*
* @param nodeId nodeId, required.
* @param applicationId applicationId. When null is specified, return
- * aggregated cardinality among all nodes.
+ * aggregated cardinality among all applications.
* @param tags allocation tags, see
* {@link SchedulingRequest#getAllocationTags()},
* When multiple tags specified. Returns cardinality
@@ -396,7 +476,7 @@ public class AllocationTagsManager {
* specified, all tags (of the node/app) will be
* considered.
* @param op operator. Such as Long::max, Long::sum, etc. Required.
- * This sparameter only take effect when #values >= 2.
+ * This parameter only take effect when #values >= 2.
* @return cardinality of specified query on the node.
* @throws InvalidAllocationTagsQueryException when illegal query
* parameter specified
@@ -414,9 +494,9 @@ public class AllocationTagsManager {
NodeToCountedTags mapping;
if (applicationId != null) {
- mapping = perAppMappings.get(applicationId);
- } else{
- mapping = globalMapping;
+ mapping = perAppNodeMappings.get(applicationId);
+ } else {
+ mapping = globalNodeMapping;
}
if (mapping == null) {
@@ -428,4 +508,52 @@ public class AllocationTagsManager {
readLock.unlock();
}
}
+
+ /**
+ * Get cardinality for following conditions. External can pass-in a binary op
+ * to implement customized logic.
+ *
+ * @param rack rack, required.
+ * @param applicationId applicationId. When null is specified, return
+ * aggregated cardinality among all applications.
+ * @param tags allocation tags, see
+ * {@link SchedulingRequest#getAllocationTags()},
+ * When multiple tags specified. Returns cardinality
+ * depends on op. If a specified tag doesn't exist, 0
+ * will be its cardinality. When null/empty tags
+ * specified, all tags (of the rack/app) will be
+ * considered.
+ * @param op operator. Such as Long::max, Long::sum, etc. Required.
+ * This parameter only take effect when #values >= 2.
+ * @return cardinality of specified query on the rack.
+ * @throws InvalidAllocationTagsQueryException when illegal query
+ * parameter specified
+ */
+ public long getRackCardinalityByOp(String rack, ApplicationId applicationId,
+ Set<String> tags, LongBinaryOperator op)
+ throws InvalidAllocationTagsQueryException {
+ readLock.lock();
+
+ try {
+ if (rack == null || op == null) {
+ throw new InvalidAllocationTagsQueryException(
+ "Must specify rack/tags/op to query cardinality");
+ }
+
+ NodeToCountedTags mapping;
+ if (applicationId != null) {
+ mapping = perAppRackMappings.get(applicationId);
+ } else {
+ mapping = globalRackMapping;
+ }
+
+ if (mapping == null) {
+ return 0;
+ }
+
+ return mapping.getCardinality(rack, tags, op);
+ } finally {
+ readLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3dd5204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 538d128..b927870 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -405,8 +405,8 @@ public class TestRMContainerImpl {
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
- AllocationTagsManager tagsManager = new AllocationTagsManager();
RMContext rmContext = mock(RMContext.class);
+ AllocationTagsManager tagsManager = new AllocationTagsManager(rmContext);
when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3dd5204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
index 4bb2a18..0ce1614 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestAllocationTagsManager.java
@@ -20,202 +20,300 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
-import com.google.common.collect.ImmutableSet;
+import java.util.List;
+
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import com.google.common.collect.ImmutableSet;
+
/**
* Test functionality of AllocationTagsManager.
*/
public class TestAllocationTagsManager {
+ private RMContext rmContext;
+
+ @Before
+ public void setup() {
+ MockRM rm = new MockRM();
+ rm.start();
+ MockNodes.resetHostIds();
+ List<RMNode> rmNodes =
+ MockNodes.newNodes(2, 4, Resource.newInstance(4096, 4));
+ for (RMNode rmNode : rmNodes) {
+ rm.getRMContext().getRMNodes().putIfAbsent(rmNode.getNodeID(), rmNode);
+ }
+ rmContext = rm.getRMContext();
+ }
+
+
@Test
public void testAllocationTagsManagerSimpleCases()
throws InvalidAllocationTagsQueryException {
- AllocationTagsManager atm = new AllocationTagsManager();
+
+ AllocationTagsManager atm = new AllocationTagsManager(rmContext);
/**
* Construct test case:
- * Node1:
+ * Node1 (rack0):
* container_1_1 (mapper/reducer/app_1)
* container_1_3 (service/app_1)
*
- * Node2:
+ * Node2 (rack0):
* container_1_2 (mapper/reducer/app_1)
* container_1_4 (reducer/app_1)
* container_2_1 (service/app_2)
*/
// 3 Containers from app1
- atm.addContainer(NodeId.fromString("node1:1234"),
+ atm.addContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("node1:1234"),
+ atm.addContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
// 1 Container from app2
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
- // Get Cardinality of app1 on node1, with tag "mapper"
+ // Get Node Cardinality of app1 on node1, with tag "mapper"
Assert.assertEquals(1,
- atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
Long::max));
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
+ // Get Rack Cardinality of app1 on rack0, with tag "mapper"
+ Assert.assertEquals(2, atm.getRackCardinality("rack0",
+ TestUtils.getMockApplicationId(1), "mapper"));
+
+ // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=min
Assert.assertEquals(1,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("mapper", "reducer"), Long::min));
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
+ // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=max
Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("mapper", "reducer"), Long::max));
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
+ // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
Assert.assertEquals(3,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("mapper", "reducer"), Long::sum));
- // Get Cardinality by passing single tag.
+ // Get Node Cardinality by passing single tag.
Assert.assertEquals(1,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinality(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), "mapper"));
Assert.assertEquals(2,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinality(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), "reducer"));
- // Get Cardinality of app1 on node2, with tag "no_existed/reducer", op=min
+ // Get Node Cardinality of app1 on node2, with tag "no_existed/reducer",
+ // op=min
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("no_existed", "reducer"), Long::min));
- // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
+ // Get Node Cardinality of app1 on node2, with tag "<applicationId>", op=max
// (Expect this returns #containers from app1 on node2)
+ Assert
+ .assertEquals(2,
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(1),
+ ImmutableSet.of(AllocationTagsNamespaces.APP_ID
+ + TestUtils.getMockApplicationId(1).toString()),
+ Long::max));
+
+ // Get Node Cardinality of app1 on node2, with empty tag set, op=max
Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
- TestUtils.getMockApplicationId(1), ImmutableSet
- .of(AllocationTagsNamespaces.APP_ID + TestUtils
- .getMockApplicationId(1).toString()), Long::max));
-
- // Get Cardinality of app1 on node2, with empty tag set, op=max
- Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
- // Get Cardinality of all apps on node2, with empty tag set, op=sum
- Assert.assertEquals(7,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
- ImmutableSet.of(), Long::sum));
+ // Get Node Cardinality of all apps on node2, with empty tag set, op=sum
+ Assert.assertEquals(7, atm.getNodeCardinalityByOp(
+ NodeId.fromString("host2:123"), null, ImmutableSet.of(), Long::sum));
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum
Assert.assertEquals(5,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum
Assert.assertEquals(2,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
// Finish all containers:
- atm.removeContainer(NodeId.fromString("node1:1234"),
+ atm.removeContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
- atm.removeContainer(NodeId.fromString("node2:1234"),
+ atm.removeContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
- atm.removeContainer(NodeId.fromString("node1:1234"),
+ atm.removeContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
- atm.removeContainer(NodeId.fromString("node2:1234"),
+ atm.removeContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
- atm.removeContainer(NodeId.fromString("node2:1234"),
+ atm.removeContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
// Expect all cardinality to be 0
// Get Cardinality of app1 on node1, with tag "mapper"
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node1:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), ImmutableSet.of("mapper"),
Long::max));
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=min
+ // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=min
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("mapper", "reducer"), Long::min));
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=max
+ // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=max
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("mapper", "reducer"), Long::max));
- // Get Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
+ // Get Node Cardinality of app1 on node2, with tag "mapper/reducer", op=sum
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of("mapper", "reducer"), Long::sum));
- // Get Cardinality of app1 on node2, with tag "<applicationId>", op=max
+ // Get Node Cardinality of app1 on node2, with tag "<applicationId>", op=max
// (Expect this returns #containers from app1 on node2)
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
ImmutableSet.of(TestUtils.getMockApplicationId(1).toString()),
Long::max));
Assert.assertEquals(0,
- atm.getNodeCardinality(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinality(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1),
TestUtils.getMockApplicationId(1).toString()));
- // Get Cardinality of app1 on node2, with empty tag set, op=max
+ // Get Node Cardinality of app1 on node2, with empty tag set, op=max
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
- // Get Cardinality of all apps on node2, with empty tag set, op=sum
- Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"), null,
- ImmutableSet.of(), Long::sum));
+ // Get Node Cardinality of all apps on node2, with empty tag set, op=sum
+ Assert.assertEquals(0, atm.getNodeCardinalityByOp(
+ NodeId.fromString("host2:123"), null, ImmutableSet.of(), Long::sum));
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ // Get Node Cardinality of app_1 on node2, with empty tag set, op=sum
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::sum));
- // Get Cardinality of app_1 on node2, with empty tag set, op=sum
+ // Get Node Cardinality of app_2 on node2, with empty tag set, op=sum
Assert.assertEquals(0,
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), ImmutableSet.of(), Long::sum));
}
+
+ @Test
+ public void testAllocationTagsManagerRackMapping()
+ throws InvalidAllocationTagsQueryException {
+
+ AllocationTagsManager atm = new AllocationTagsManager(rmContext);
+
+ /**
+ * Construct Rack test case:
+ * Node1 (rack0):
+ * container_1_1 (mapper/reducer/app_1)
+ * container_1_4 (reducer/app_2)
+ *
+ * Node2 (rack0):
+ * container_1_2 (mapper/reducer/app_2)
+ * container_1_3 (service/app_1)
+ *
+ * Node5 (rack1):
+ * container_2_1 (service/app_2)
+ */
+
+ // 3 Containers from app1
+ atm.addContainer(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 2),
+ ImmutableSet.of("mapper", "reducer"));
+
+ atm.addContainer(NodeId.fromString("host1:123"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 4),
+ ImmutableSet.of("reducer"));
+
+ atm.addContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
+ ImmutableSet.of("service"));
+
+ // 1 Container from app2
+ atm.addContainer(NodeId.fromString("host2:123"),
+ TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
+ ImmutableSet.of("service"));
+
+ // Get Rack Cardinality of app1 on rack0, with tag "mapper"
+ Assert.assertEquals(1, atm.getRackCardinality("rack0",
+ TestUtils.getMockApplicationId(1), "mapper"));
+
+ // Get Rack Cardinality of app2 on rack0, with tag "reducer"
+ Assert.assertEquals(2, atm.getRackCardinality("rack0",
+ TestUtils.getMockApplicationId(2), "reducer"));
+
+ // Get Rack Cardinality of all apps on rack0, with tag "reducer"
+ Assert.assertEquals(3, atm.getRackCardinality("rack0", null, "reducer"));
+
+ // Get Rack Cardinality of app_1 on rack0, with empty tag set, op=max
+ Assert.assertEquals(2, atm.getRackCardinalityByOp("rack0",
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::max));
+
+ // Get Rack Cardinality of app_1 on rack0, with empty tag set, op=min
+ Assert.assertEquals(1, atm.getRackCardinalityByOp("rack0",
+ TestUtils.getMockApplicationId(1), ImmutableSet.of(), Long::min));
+
+ // Get Rack Cardinality of all apps on rack0, with empty tag set, op=min
+ Assert.assertEquals(3, atm.getRackCardinalityByOp("rack0", null,
+ ImmutableSet.of(), Long::max));
+ }
+
@Test
public void testAllocationTagsManagerMemoryAfterCleanup()
throws InvalidAllocationTagsQueryException {
@@ -223,54 +321,57 @@ public class TestAllocationTagsManager {
* Make sure YARN cleans up all memory once container/app finishes.
*/
- AllocationTagsManager atm = new AllocationTagsManager();
+ AllocationTagsManager atm = new AllocationTagsManager(rmContext);
// Add a bunch of containers
- atm.addContainer(NodeId.fromString("node1:1234"),
+ atm.addContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("node1:1234"),
+ atm.addContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
// Remove all these containers
- atm.removeContainer(NodeId.fromString("node1:1234"),
+ atm.removeContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
- atm.removeContainer(NodeId.fromString("node2:1234"),
+ atm.removeContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
- atm.removeContainer(NodeId.fromString("node1:1234"),
+ atm.removeContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
- atm.removeContainer(NodeId.fromString("node2:1234"),
+ atm.removeContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
- atm.removeContainer(NodeId.fromString("node2:1234"),
+ atm.removeContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
// Check internal data structure
Assert.assertEquals(0,
- atm.getGlobalMapping().getNodeToTagsWithCount().size());
- Assert.assertEquals(0, atm.getPerAppMappings().size());
+ atm.getGlobalNodeMapping().getTypeToTagsWithCount().size());
+ Assert.assertEquals(0, atm.getPerAppNodeMappings().size());
+ Assert.assertEquals(0,
+ atm.getGlobalRackMapping().getTypeToTagsWithCount().size());
+ Assert.assertEquals(0, atm.getPerAppRackMappings().size());
}
@Test
@@ -280,26 +381,26 @@ public class TestAllocationTagsManager {
* Make sure YARN cleans up all memory once container/app finishes.
*/
- AllocationTagsManager atm = new AllocationTagsManager();
+ AllocationTagsManager atm = new AllocationTagsManager(rmContext);
// Add a bunch of containers
- atm.addContainer(NodeId.fromString("node1:1234"),
+ atm.addContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 1),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 2),
ImmutableSet.of("mapper", "reducer"));
- atm.addContainer(NodeId.fromString("node1:1234"),
+ atm.addContainer(NodeId.fromString("host1:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 3),
ImmutableSet.of("service"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(1), TestUtils.getMockContainerId(1, 4),
ImmutableSet.of("reducer"));
- atm.addContainer(NodeId.fromString("node2:1234"),
+ atm.addContainer(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), TestUtils.getMockContainerId(2, 3),
ImmutableSet.of("service"));
@@ -317,7 +418,7 @@ public class TestAllocationTagsManager {
// No op
caughtException = false;
try {
- atm.getNodeCardinalityByOp(NodeId.fromString("node2:1234"),
+ atm.getNodeCardinalityByOp(NodeId.fromString("host2:123"),
TestUtils.getMockApplicationId(2), ImmutableSet.of("mapper"), null);
} catch (InvalidAllocationTagsQueryException e) {
caughtException = true;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/11] hadoop git commit: YARN-6619. AMRMClient Changes to use the
PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda)
Posted by as...@apache.org.
YARN-6619. AMRMClient Changes to use the PlacementConstraint and SchcedulingRequest objects. (Arun Suresh via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4582e7b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4582e7b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4582e7b3
Branch: refs/heads/YARN-6592
Commit: 4582e7b389a811dcad399978e4e69c42a74e696b
Parents: 308fc39
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Jan 17 11:36:26 2018 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:53:52 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/client/api/AMRMClient.java | 38 +++-
.../yarn/client/api/async/AMRMClientAsync.java | 48 +++++
.../api/async/impl/AMRMClientAsyncImpl.java | 49 ++++-
.../yarn/client/api/impl/AMRMClientImpl.java | 142 ++++++++++++-
.../client/api/impl/BaseAMRMClientTest.java | 212 +++++++++++++++++++
.../yarn/client/api/impl/TestAMRMClient.java | 156 +-------------
.../TestAMRMClientPlacementConstraints.java | 204 ++++++++++++++++++
.../rmcontainer/RMContainerImpl.java | 3 +
.../scheduler/AbstractYarnScheduler.java | 1 +
.../scheduler/SchedulerApplicationAttempt.java | 1 +
.../constraint/PlacementConstraintsUtil.java | 4 +-
11 files changed, 700 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index d3d1974..914a146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api;
import java.io.IOException;
import java.util.Collection;
+import java.util.Map;
+import java.util.Set;
import java.util.function.Supplier;
import java.util.List;
@@ -39,7 +41,9 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -554,6 +558,18 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
}
/**
+ * Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ * all requests in the same batch are sent in the same allocate call.
+ * @param schedulingRequests Collection of Scheduling Requests.
+ */
+ @Public
+ @InterfaceStability.Unstable
+ public void addSchedulingRequests(
+ Collection<SchedulingRequest> schedulingRequests) {
+
+ }
+
+ /**
* Register the application master. This must be called before any
* other interaction
* @param appHostName Name of the host on which master is running
@@ -568,7 +584,27 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
int appHostPort,
String appTrackingUrl)
throws YarnException, IOException;
-
+
+ /**
+ * Register the application master. This must be called before any
+ * other interaction
+ * @param appHostName Name of the host on which master is running
+ * @param appHostPort Port master is listening on
+ * @param appTrackingUrl URL at which the master info can be seen
+ * @param placementConstraints Placement Constraints mappings.
+ * @return <code>RegisterApplicationMasterResponse</code>
+ * @throws YarnException
+ * @throws IOException
+ */
+ @Public
+ @InterfaceStability.Unstable
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl,
+ Map<Set<String>, PlacementConstraint> placementConstraints)
+ throws YarnException, IOException {
+ throw new YarnException("Not supported");
+ }
+
/**
* Request additional containers and receive new container allocations.
* Requests made via <code>addContainerRequest</code> are sent to the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index 2b82ad6..0af687b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.client.api.async;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
@@ -38,9 +40,12 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
@@ -206,6 +211,19 @@ extends AbstractService {
Resource capability);
/**
+ * Add a Collection of SchedulingRequests. The AMRMClient will ensure that
+ * all requests in the same batch are sent in the same allocate call.
+ * @param schedulingRequests Collection of Scheduling Requests.
+ */
+ @Public
+ @Unstable
+ public void addSchedulingRequests(
+ Collection<SchedulingRequest> schedulingRequests) {
+
+ }
+
+
+ /**
* Returns all matching ContainerRequests that match the given Priority,
* ResourceName, ExecutionType and Capability.
*
@@ -250,6 +268,26 @@ extends AbstractService {
throws YarnException, IOException;
/**
+ * Register the application master. This must be called before any
+ * other interaction
+ * @param appHostName Name of the host on which master is running
+ * @param appHostPort Port master is listening on
+ * @param appTrackingUrl URL at which the master info can be seen
+ * @param placementConstraints Placement Constraints mappings.
+ * @return <code>RegisterApplicationMasterResponse</code>
+ * @throws YarnException
+ * @throws IOException
+ */
+ @Public
+ @Unstable
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl,
+ Map<Set<String>, PlacementConstraint> placementConstraints)
+ throws YarnException, IOException {
+ throw new YarnException("Not supported");
+ }
+
+ /**
* Unregister the application master. This must be called in the end.
* @param appStatus Success/Failure status of the master
* @param appMessage Diagnostics message on failure
@@ -494,6 +532,16 @@ extends AbstractService {
public void onContainersReceivedFromPreviousAttempts(
List<Container> containers) {
}
+
+ /**
+ * Called when the RM has rejected Scheduling Requests.
+ * @param rejectedSchedulingRequests Rejected Scheduling Requests.
+ */
+ @Public
+ @Unstable
+ public void onRequestsRejected(
+ List<RejectedSchedulingRequest> rejectedSchedulingRequests) {
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 33b0aba..4f04b66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -36,9 +38,12 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.TimelineV2Client;
@@ -150,18 +155,50 @@ extends AMRMClientAsync<T> {
Resource capability) {
return client.getMatchingRequests(priority, resourceName, capability);
}
-
+
+ @Override
+ public void addSchedulingRequests(
+ Collection<SchedulingRequest> schedulingRequests) {
+ client.addSchedulingRequests(schedulingRequests);
+ }
+
/**
* Registers this application master with the resource manager. On successful
* registration, starts the heartbeating thread.
+ *
+ * @param appHostName Name of the host on which master is running
+ * @param appHostPort Port master is listening on
+ * @param appTrackingUrl URL at which the master info can be seen
+ * @return Register AM Response.
* @throws YarnException
* @throws IOException
*/
public RegisterApplicationMasterResponse registerApplicationMaster(
String appHostName, int appHostPort, String appTrackingUrl)
throws YarnException, IOException {
+ return registerApplicationMaster(
+ appHostName, appHostPort, appTrackingUrl, null);
+ }
+
+ /**
+ * Registers this application master with the resource manager. On successful
+ * registration, starts the heartbeating thread.
+ *
+ * @param appHostName Name of the host on which master is running
+ * @param appHostPort Port master is listening on
+ * @param appTrackingUrl URL at which the master info can be seen
+ * @param placementConstraintsMap Placement Constraints Mapping.
+ * @return Register AM Response.
+ * @throws YarnException
+ * @throws IOException
+ */
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl,
+ Map<Set<String>, PlacementConstraint> placementConstraintsMap)
+ throws YarnException, IOException {
RegisterApplicationMasterResponse response = client
- .registerApplicationMaster(appHostName, appHostPort, appTrackingUrl);
+ .registerApplicationMaster(appHostName, appHostPort,
+ appTrackingUrl, placementConstraintsMap);
heartbeatThread.start();
return response;
}
@@ -366,6 +403,14 @@ extends AMRMClientAsync<T> {
response.getContainersFromPreviousAttempts());
}
}
+ List<RejectedSchedulingRequest> rejectedSchedulingRequests =
+ response.getRejectedSchedulingRequests();
+ if (!rejectedSchedulingRequests.isEmpty()) {
+ if (handler instanceof AMRMClientAsync.AbstractCallbackHandler) {
+ ((AMRMClientAsync.AbstractCallbackHandler) handler)
+ .onRequestsRejected(rejectedSchedulingRequests);
+ }
+ }
progress = handler.getProgress();
} catch (Throwable ex) {
handler.onError(ex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 5507c07..8e2336f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -30,9 +30,11 @@ import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
import java.util.AbstractMap.SimpleEntry;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -60,9 +62,11 @@ import org.apache.hadoop.yarn.api.records.ProfileCapability;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -106,6 +110,12 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
protected final Set<String> blacklistedNodes = new HashSet<String>();
protected final Set<String> blacklistAdditions = new HashSet<String>();
protected final Set<String> blacklistRemovals = new HashSet<String>();
+ private Map<Set<String>, PlacementConstraint> placementConstraints =
+ new HashMap<>();
+ private Queue<Collection<SchedulingRequest>> batchedSchedulingRequests =
+ new LinkedList<>();
+ private Map<Set<String>, List<SchedulingRequest>> outstandingSchedRequests =
+ new ConcurrentHashMap<>();
protected Map<String, Resource> resourceProfilesMap;
@@ -218,14 +228,26 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
super.serviceStop();
}
-
+
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(
String appHostName, int appHostPort, String appTrackingUrl)
throws YarnException, IOException {
+ return registerApplicationMaster(appHostName, appHostPort, appTrackingUrl,
+ null);
+ }
+
+ @Override
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl,
+ Map<Set<String>, PlacementConstraint> placementConstraintsMap)
+ throws YarnException, IOException {
this.appHostName = appHostName;
this.appHostPort = appHostPort;
this.appTrackingUrl = appTrackingUrl;
+ if (placementConstraintsMap != null && !placementConstraintsMap.isEmpty()) {
+ this.placementConstraints.putAll(placementConstraintsMap);
+ }
Preconditions.checkArgument(appHostName != null,
"The host name should not be null");
Preconditions.checkArgument(appHostPort >= -1, "Port number of the host"
@@ -240,6 +262,9 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
RegisterApplicationMasterRequest request =
RegisterApplicationMasterRequest.newInstance(this.appHostName,
this.appHostPort, this.appTrackingUrl);
+ if (!this.placementConstraints.isEmpty()) {
+ request.setPlacementConstraints(this.placementConstraints);
+ }
RegisterApplicationMasterResponse response =
rmClient.registerApplicationMaster(request);
synchronized (this) {
@@ -248,11 +273,23 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
populateNMTokens(response.getNMTokensFromPreviousAttempts());
}
this.resourceProfilesMap = response.getResourceProfiles();
+ List<Container> prevContainers =
+ response.getContainersFromPreviousAttempts();
+ removeFromOutstandingSchedulingRequests(prevContainers);
+ recreateSchedulingRequestBatch();
}
return response;
}
@Override
+ public void addSchedulingRequests(
+ Collection<SchedulingRequest> schedulingRequests) {
+ synchronized (this.batchedSchedulingRequests) {
+ this.batchedSchedulingRequests.add(schedulingRequests);
+ }
+ }
+
+ @Override
public AllocateResponse allocate(float progressIndicator)
throws YarnException, IOException {
Preconditions.checkArgument(progressIndicator >= 0,
@@ -288,6 +325,7 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
.responseId(lastResponseId).progress(progressIndicator)
.askList(askList).resourceBlacklistRequest(blacklistRequest)
.releaseList(releaseList).updateRequests(updateList).build();
+ populateSchedulingRequests(allocateRequest);
// clear blacklistAdditions and blacklistRemovals before
// unsynchronized part
blacklistAdditions.clear();
@@ -296,6 +334,10 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
try {
allocateResponse = rmClient.allocate(allocateRequest);
+ removeFromOutstandingSchedulingRequests(
+ allocateResponse.getAllocatedContainers());
+ removeFromOutstandingSchedulingRequests(
+ allocateResponse.getContainersFromPreviousAttempts());
} catch (ApplicationMasterNotRegisteredException e) {
LOG.warn("ApplicationMaster is out of sync with ResourceManager,"
+ " hence resyncing.");
@@ -397,6 +439,104 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
return allocateResponse;
}
+ private void populateSchedulingRequests(AllocateRequest allocateRequest) {
+ synchronized (this.batchedSchedulingRequests) {
+ if (!this.batchedSchedulingRequests.isEmpty()) {
+ List<SchedulingRequest> newReqs = new LinkedList<>();
+ Iterator<Collection<SchedulingRequest>> iter =
+ this.batchedSchedulingRequests.iterator();
+ while (iter.hasNext()) {
+ Collection<SchedulingRequest> requests = iter.next();
+ newReqs.addAll(requests);
+ addToOutstandingSchedulingRequests(requests);
+ iter.remove();
+ }
+ allocateRequest.setSchedulingRequests(newReqs);
+ }
+ }
+ }
+
+ private void recreateSchedulingRequestBatch() {
+ List<SchedulingRequest> batched = new ArrayList<>();
+ synchronized (this.outstandingSchedRequests) {
+ for (List<SchedulingRequest> schedReqs :
+ this.outstandingSchedRequests.values()) {
+ batched.addAll(schedReqs);
+ }
+ }
+ synchronized (this.batchedSchedulingRequests) {
+ this.batchedSchedulingRequests.add(batched);
+ }
+ }
+
+ private void addToOutstandingSchedulingRequests(
+ Collection<SchedulingRequest> requests) {
+ for (SchedulingRequest req : requests) {
+ List<SchedulingRequest> schedulingRequests =
+ this.outstandingSchedRequests.computeIfAbsent(
+ req.getAllocationTags(), x -> new LinkedList<>());
+ SchedulingRequest matchingReq = null;
+ synchronized (schedulingRequests) {
+ for (SchedulingRequest schedReq : schedulingRequests) {
+ if (isMatching(req, schedReq)) {
+ matchingReq = schedReq;
+ break;
+ }
+ }
+ if (matchingReq != null) {
+ matchingReq.getResourceSizing().setNumAllocations(
+ req.getResourceSizing().getNumAllocations());
+ } else {
+ schedulingRequests.add(req);
+ }
+ }
+ }
+ }
+
+ private boolean isMatching(SchedulingRequest schedReq1,
+ SchedulingRequest schedReq2) {
+ return schedReq1.getPriority().equals(schedReq2.getPriority()) &&
+ schedReq1.getExecutionType().getExecutionType().equals(
+ schedReq1.getExecutionType().getExecutionType()) &&
+ schedReq1.getAllocationRequestId() ==
+ schedReq2.getAllocationRequestId();
+ }
+
+ private void removeFromOutstandingSchedulingRequests(
+ Collection<Container> containers) {
+ if (containers == null || containers.isEmpty()) {
+ return;
+ }
+ for (Container container : containers) {
+ if (container.getAllocationTags() != null &&
+ !container.getAllocationTags().isEmpty()) {
+ List<SchedulingRequest> schedReqs =
+ this.outstandingSchedRequests.get(container.getAllocationTags());
+ if (schedReqs != null && !schedReqs.isEmpty()) {
+ synchronized (schedReqs) {
+ Iterator<SchedulingRequest> iter = schedReqs.iterator();
+ while (iter.hasNext()) {
+ SchedulingRequest schedReq = iter.next();
+ if (schedReq.getPriority().equals(container.getPriority()) &&
+ schedReq.getAllocationRequestId() ==
+ container.getAllocationRequestId()) {
+ int numAllocations =
+ schedReq.getResourceSizing().getNumAllocations();
+ numAllocations--;
+ if (numAllocations == 0) {
+ iter.remove();
+ } else {
+ schedReq.getResourceSizing()
+ .setNumAllocations(numAllocations);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
private List<UpdateContainerRequest> createUpdateList() {
List<UpdateContainerRequest> updateList = new ArrayList<>();
for (Map.Entry<ContainerId, SimpleEntry<Container,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/BaseAMRMClientTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/BaseAMRMClientTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/BaseAMRMClientTest.java
new file mode 100644
index 0000000..d18652f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/BaseAMRMClientTest.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Base class for testing AMRMClient.
+ */
+public class BaseAMRMClientTest {
+
+ protected Configuration conf = null;
+ protected MiniYARNCluster yarnCluster = null;
+ protected YarnClient yarnClient = null;
+ protected List<NodeReport> nodeReports = null;
+ protected ApplicationAttemptId attemptId = null;
+
+ protected String schedulerName = CapacityScheduler.class.getName();
+ protected boolean autoUpdate = false;
+
+ protected int nodeCount = 3;
+ protected long amExpireMs = 4000;
+ protected int rollingIntervalSec = 13;
+
+
+ protected Resource capability;
+ protected Priority priority;
+ protected Priority priority2;
+ protected String node;
+ protected String rack;
+ protected String[] nodes;
+ protected String[] racks;
+
+ @Before
+ public void setup() throws Exception {
+ conf = new YarnConfiguration();
+ createClusterAndStartApplication(conf);
+ }
+
+ protected void createClusterAndStartApplication(Configuration conf)
+ throws Exception {
+ // start minicluster
+ this.conf = conf;
+ if (autoUpdate) {
+ conf.setBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS, true);
+ }
+ conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName);
+ conf.setLong(
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ rollingIntervalSec);
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, amExpireMs);
+ conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
+ // set the minimum allocation so that resource decrease can go under 1024
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
+ conf.setBoolean(
+ YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH, 10);
+ yarnCluster = new MiniYARNCluster(
+ TestAMRMClient.class.getName(), nodeCount, 1, 1);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+
+ // start rm client
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(conf);
+ yarnClient.start();
+
+ // get node info
+ assertTrue("All node managers did not connect to the RM within the "
+ + "allotted 5-second timeout",
+ yarnCluster.waitForNodeManagersToConnect(5000L));
+ nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
+ assertEquals("Not all node managers were reported running",
+ nodeCount, nodeReports.size());
+
+ priority = Priority.newInstance(1);
+ priority2 = Priority.newInstance(2);
+ capability = Resource.newInstance(1024, 1);
+
+ node = nodeReports.get(0).getNodeId().getHost();
+ rack = nodeReports.get(0).getRackName();
+ nodes = new String[]{ node };
+ racks = new String[]{ rack };
+
+ // submit new app
+ ApplicationSubmissionContext appContext =
+ yarnClient.createApplication().getApplicationSubmissionContext();
+ ApplicationId appId = appContext.getApplicationId();
+ // set the application name
+ appContext.setApplicationName("Test");
+ // Set the priority for the application master
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(0);
+ appContext.setPriority(pri);
+ // Set the queue to which this application is to be submitted in the RM
+ appContext.setQueue("default");
+ // Set up the container launch context for the application master
+ ContainerLaunchContext amContainer =
+ BuilderUtils.newContainerLaunchContext(
+ Collections.<String, LocalResource> emptyMap(),
+ new HashMap<String, String>(), Arrays.asList("sleep", "100"),
+ new HashMap<String, ByteBuffer>(), null,
+ new HashMap<ApplicationAccessType, String>());
+ appContext.setAMContainerSpec(amContainer);
+ appContext.setResource(Resource.newInstance(1024, 1));
+ // Create the request to send to the applications manager
+ SubmitApplicationRequest appRequest = Records
+ .newRecord(SubmitApplicationRequest.class);
+ appRequest.setApplicationSubmissionContext(appContext);
+ // Submit the application to the applications manager
+ yarnClient.submitApplication(appContext);
+
+ // wait for app to start
+ RMAppAttempt appAttempt = null;
+ while (true) {
+ ApplicationReport appReport = yarnClient.getApplicationReport(appId);
+ if (appReport.getYarnApplicationState() ==
+ YarnApplicationState.ACCEPTED) {
+ attemptId = appReport.getCurrentApplicationAttemptId();
+ appAttempt =
+ yarnCluster.getResourceManager().getRMContext().getRMApps()
+ .get(attemptId.getApplicationId()).getCurrentAppAttempt();
+ while (true) {
+ if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
+ break;
+ }
+ }
+ break;
+ }
+ }
+ // Just dig into the ResourceManager and get the AMRMToken just for the sake
+ // of testing.
+ UserGroupInformation.setLoginUser(UserGroupInformation
+ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
+
+ // emulate RM setup of AMRM token in credentials by adding the token
+ // *before* setting the token service
+ UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
+ appAttempt.getAMRMToken().setService(
+ ClientRMProxy.getAMRMTokenService(conf));
+ }
+
+ @After
+ public void teardown() throws YarnException, IOException {
+ yarnClient.killApplication(attemptId.getApplicationId());
+ attemptId = null;
+
+ if (yarnClient != null &&
+ yarnClient.getServiceState() == Service.STATE.STARTED) {
+ yarnClient.stop();
+ }
+ if (yarnCluster != null &&
+ yarnCluster.getServiceState() == Service.STATE.STARTED) {
+ yarnCluster.stop();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 3ecc5cd..b059118 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -43,7 +43,6 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
@@ -56,24 +55,18 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.*;
-import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
-import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -81,10 +74,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedule
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
-import org.junit.After;
import org.junit.Assert;
import org.junit.Assume;
-import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -97,26 +88,8 @@ import org.eclipse.jetty.util.log.Log;
* Test application master client class to resource manager.
*/
@RunWith(value = Parameterized.class)
-public class TestAMRMClient {
- private String schedulerName = null;
- private boolean autoUpdate = false;
- private Configuration conf = null;
- private MiniYARNCluster yarnCluster = null;
- private YarnClient yarnClient = null;
- private List<NodeReport> nodeReports = null;
- private ApplicationAttemptId attemptId = null;
- private int nodeCount = 3;
-
- static final int rolling_interval_sec = 13;
- static final long am_expire_ms = 4000;
-
- private Resource capability;
- private Priority priority;
- private Priority priority2;
- private String node;
- private String rack;
- private String[] nodes;
- private String[] racks;
+public class TestAMRMClient extends BaseAMRMClientTest{
+
private final static int DEFAULT_ITERATION = 3;
public TestAMRMClient(String schedulerName, boolean autoUpdate) {
@@ -134,127 +107,6 @@ public class TestAMRMClient {
});
}
- @Before
- public void setup() throws Exception {
- conf = new YarnConfiguration();
- createClusterAndStartApplication(conf);
- }
-
- private void createClusterAndStartApplication(Configuration conf)
- throws Exception {
- // start minicluster
- this.conf = conf;
- if (autoUpdate) {
- conf.setBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS, true);
- }
- conf.set(YarnConfiguration.RM_SCHEDULER, schedulerName);
- conf.setLong(
- YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
- rolling_interval_sec);
- conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
- conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
- // set the minimum allocation so that resource decrease can go under 1024
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
- conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
- conf.setBoolean(
- YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
- conf.setInt(
- YarnConfiguration.NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH, 10);
- yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
- yarnCluster.init(conf);
- yarnCluster.start();
-
- // start rm client
- yarnClient = YarnClient.createYarnClient();
- yarnClient.init(conf);
- yarnClient.start();
-
- // get node info
- assertTrue("All node managers did not connect to the RM within the "
- + "allotted 5-second timeout",
- yarnCluster.waitForNodeManagersToConnect(5000L));
- nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
- assertEquals("Not all node managers were reported running",
- nodeCount, nodeReports.size());
-
- priority = Priority.newInstance(1);
- priority2 = Priority.newInstance(2);
- capability = Resource.newInstance(1024, 1);
-
- node = nodeReports.get(0).getNodeId().getHost();
- rack = nodeReports.get(0).getRackName();
- nodes = new String[]{ node };
- racks = new String[]{ rack };
-
- // submit new app
- ApplicationSubmissionContext appContext =
- yarnClient.createApplication().getApplicationSubmissionContext();
- ApplicationId appId = appContext.getApplicationId();
- // set the application name
- appContext.setApplicationName("Test");
- // Set the priority for the application master
- Priority pri = Records.newRecord(Priority.class);
- pri.setPriority(0);
- appContext.setPriority(pri);
- // Set the queue to which this application is to be submitted in the RM
- appContext.setQueue("default");
- // Set up the container launch context for the application master
- ContainerLaunchContext amContainer =
- BuilderUtils.newContainerLaunchContext(
- Collections.<String, LocalResource> emptyMap(),
- new HashMap<String, String>(), Arrays.asList("sleep", "100"),
- new HashMap<String, ByteBuffer>(), null,
- new HashMap<ApplicationAccessType, String>());
- appContext.setAMContainerSpec(amContainer);
- appContext.setResource(Resource.newInstance(1024, 1));
- // Create the request to send to the applications manager
- SubmitApplicationRequest appRequest = Records
- .newRecord(SubmitApplicationRequest.class);
- appRequest.setApplicationSubmissionContext(appContext);
- // Submit the application to the applications manager
- yarnClient.submitApplication(appContext);
-
- // wait for app to start
- RMAppAttempt appAttempt = null;
- while (true) {
- ApplicationReport appReport = yarnClient.getApplicationReport(appId);
- if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
- attemptId = appReport.getCurrentApplicationAttemptId();
- appAttempt =
- yarnCluster.getResourceManager().getRMContext().getRMApps()
- .get(attemptId.getApplicationId()).getCurrentAppAttempt();
- while (true) {
- if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
- break;
- }
- }
- break;
- }
- }
- // Just dig into the ResourceManager and get the AMRMToken just for the sake
- // of testing.
- UserGroupInformation.setLoginUser(UserGroupInformation
- .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
-
- // emulate RM setup of AMRM token in credentials by adding the token
- // *before* setting the token service
- UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
- appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
- }
-
- @After
- public void teardown() throws YarnException, IOException {
- yarnClient.killApplication(attemptId.getApplicationId());
- attemptId = null;
-
- if (yarnClient != null && yarnClient.getServiceState() == STATE.STARTED) {
- yarnClient.stop();
- }
- if (yarnCluster != null && yarnCluster.getServiceState() == STATE.STARTED) {
- yarnCluster.stop();
- }
- }
-
@Test (timeout = 60000)
public void testAMRMClientNoMatchingRequests()
throws IOException, YarnException {
@@ -905,7 +757,7 @@ public class TestAMRMClient {
initAMRMClientAndTest(false);
}
- private void initAMRMClientAndTest(boolean useAllocReqId)
+ protected void initAMRMClientAndTest(boolean useAllocReqId)
throws YarnException, IOException {
AMRMClient<ContainerRequest> amClient = null;
try {
@@ -1946,7 +1798,7 @@ public class TestAMRMClient {
// Wait for enough time and make sure the roll_over happens
// At mean time, the old AMRMToken should continue to work
while (System.currentTimeMillis() - startTime <
- rolling_interval_sec * 1000) {
+ rollingIntervalSec * 1000) {
amClient.allocate(0.1f);
sleep(1000);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java
new file mode 100644
index 0000000..fdc8d58
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientPlacementConstraints.java
@@ -0,0 +1,204 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api.impl;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.NMTokenCache;
+import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
+import org.apache.hadoop.yarn.client.api.async.impl.AMRMClientAsyncImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.lang.Thread.sleep;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+/**
+ * Test Placement Constraints and Scheduling Requests.
+ */
+public class TestAMRMClientPlacementConstraints extends BaseAMRMClientTest {
+
+ @Test(timeout=60000)
+ public void testAMRMClientWithPlacementConstraints()
+ throws Exception {
+ // we have to create a new instance of MiniYARNCluster to avoid SASL qop
+ // mismatches between client and server
+ teardown();
+ conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_ENABLED, true);
+ createClusterAndStartApplication(conf);
+
+ AMRMClient<AMRMClient.ContainerRequest> amClient =
+ AMRMClient.<AMRMClient.ContainerRequest>createAMRMClient();
+ amClient.setNMTokenCache(new NMTokenCache());
+ //asserting we are not using the singleton instance cache
+ Assert.assertNotSame(NMTokenCache.getSingleton(),
+ amClient.getNMTokenCache());
+
+ final List<Container> allocatedContainers = new ArrayList<>();
+ final List<RejectedSchedulingRequest> rejectedSchedulingRequests =
+ new ArrayList<>();
+ AMRMClientAsync asyncClient = new AMRMClientAsyncImpl<>(amClient, 1000,
+ new AMRMClientAsync.AbstractCallbackHandler() {
+ @Override
+ public void onContainersAllocated(List<Container> containers) {
+ allocatedContainers.addAll(containers);
+ }
+
+ @Override
+ public void onRequestsRejected(
+ List<RejectedSchedulingRequest> rejReqs) {
+ rejectedSchedulingRequests.addAll(rejReqs);
+ }
+
+ @Override
+ public void onContainersCompleted(List<ContainerStatus> statuses) {}
+ @Override
+ public void onContainersUpdated(List<UpdatedContainer> containers) {}
+ @Override
+ public void onShutdownRequest() {}
+ @Override
+ public void onNodesUpdated(List<NodeReport> updatedNodes) {}
+ @Override
+ public void onError(Throwable e) {}
+
+ @Override
+ public float getProgress() {
+ return 0.1f;
+ }
+ });
+
+ asyncClient.init(conf);
+ asyncClient.start();
+ Map<Set<String>, PlacementConstraint> pcMapping = new HashMap<>();
+ pcMapping.put(Collections.singleton("foo"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetNotIn(NODE, allocationTag("foo"))));
+ pcMapping.put(Collections.singleton("bar"),
+ PlacementConstraints.build(
+ PlacementConstraints.targetNotIn(NODE, allocationTag("bar"))));
+ asyncClient.registerApplicationMaster("Host", 10000, "", pcMapping);
+
+ // Send two types of requests - 4 with source tag "foo" have numAlloc = 1
+ // and 1 with source tag "bar" and has numAlloc = 4. Both should be
+ // handled similarly. i.e: Since there are only 3 nodes,
+ // 2 schedulingRequests - 1 with source tag "foo" on one with source
+ // tag "bar" should get rejected.
+ asyncClient.addSchedulingRequests(
+ Arrays.asList(
+ // 4 reqs with numAlloc = 1
+ schedulingRequest(1, 1, 1, 1, 512, "foo"),
+ schedulingRequest(1, 1, 2, 1, 512, "foo"),
+ schedulingRequest(1, 1, 3, 1, 512, "foo"),
+ schedulingRequest(1, 1, 4, 1, 512, "foo"),
+ // 1 req with numAlloc = 4
+ schedulingRequest(4, 1, 5, 1, 512, "bar")));
+
+ // kick the scheduler
+ waitForContainerAllocation(allocatedContainers,
+ rejectedSchedulingRequests, 6, 2);
+
+ Assert.assertEquals(6, allocatedContainers.size());
+ Map<NodeId, List<Container>> containersPerNode =
+ allocatedContainers.stream().collect(
+ Collectors.groupingBy(Container::getNodeId));
+
+ // Ensure 2 containers allocated per node.
+ // Each node should have a "foo" and a "bar" container.
+ Assert.assertEquals(3, containersPerNode.entrySet().size());
+ HashSet<String> srcTags = new HashSet<>(Arrays.asList("foo", "bar"));
+ containersPerNode.entrySet().forEach(
+ x ->
+ Assert.assertEquals(
+ srcTags,
+ x.getValue()
+ .stream()
+ .map(y -> y.getAllocationTags().iterator().next())
+ .collect(Collectors.toSet()))
+ );
+
+ // Ensure 2 rejected requests - 1 of "foo" and 1 of "bar"
+ Assert.assertEquals(2, rejectedSchedulingRequests.size());
+ Assert.assertEquals(srcTags,
+ rejectedSchedulingRequests
+ .stream()
+ .map(x -> x.getRequest().getAllocationTags().iterator().next())
+ .collect(Collectors.toSet()));
+
+ asyncClient.stop();
+ }
+
+ private static void waitForContainerAllocation(
+ List<Container> allocatedContainers,
+ List<RejectedSchedulingRequest> rejectedRequests,
+ int containerNum, int rejNum) throws Exception {
+
+ int maxCount = 10;
+ while (maxCount >= 0 &&
+ (allocatedContainers.size() < containerNum ||
+ rejectedRequests.size() < rejNum)) {
+ maxCount--;
+ sleep(1000);
+ }
+ }
+
+ private static SchedulingRequest schedulingRequest(int numAllocations,
+ int priority, long allocReqId, int cores, int mem, String... tags) {
+ return schedulingRequest(numAllocations, priority, allocReqId, cores, mem,
+ ExecutionType.GUARANTEED, tags);
+ }
+
+ private static SchedulingRequest schedulingRequest(int numAllocations,
+ int priority, long allocReqId, int cores, int mem,
+ ExecutionType execType, String... tags) {
+ return SchedulingRequest.newBuilder()
+ .priority(Priority.newInstance(priority))
+ .allocationRequestId(allocReqId)
+ .allocationTags(new HashSet<>(Arrays.asList(tags)))
+ .executionType(ExecutionTypeRequest.newInstance(execType, true))
+ .resourceSizing(
+ ResourceSizing.newInstance(numAllocations,
+ Resource.newInstance(mem, cores)))
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 563df0d..a504221 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -262,6 +262,9 @@ public class RMContainerImpl implements RMContainer {
rmContext.getSystemMetricsPublisher().containerCreated(
this, this.creationTime);
}
+ if (this.container != null) {
+ this.allocationTags = this.container.getAllocationTags();
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 1589d84..74456f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -590,6 +590,7 @@ public abstract class AbstractYarnScheduler
container.setVersion(status.getVersion());
container.setExecutionType(status.getExecutionType());
container.setAllocationRequestId(status.getAllocationRequestId());
+ container.setAllocationTags(status.getAllocationTags());
ApplicationAttemptId attemptId =
container.getId().getApplicationAttemptId();
RMContainer rmContainer = new RMContainerImpl(container,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 88a9049..3930a35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -672,6 +672,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
containerType, container.getExecutionType(),
container.getAllocationRequestId(),
rmContainer.getAllocationTags()));
+ container.setAllocationTags(rmContainer.getAllocationTags());
updateNMToken(container);
} catch (IllegalArgumentException e) {
// DNS might be down, skip returning this container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4582e7b3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
index 956a3c9..c4b82e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintsUtil.java
@@ -64,12 +64,12 @@ public final class PlacementConstraintsUtil {
throws InvalidAllocationTagsQueryException {
long minScopeCardinality = 0;
long maxScopeCardinality = 0;
- if (sc.getScope() == PlacementConstraints.NODE) {
+ if (sc.getScope().equals(PlacementConstraints.NODE)) {
minScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
te.getTargetValues(), Long::max);
maxScopeCardinality = tm.getNodeCardinalityByOp(node.getNodeID(), appId,
te.getTargetValues(), Long::min);
- } else if (sc.getScope() == PlacementConstraints.RACK) {
+ } else if (sc.getScope().equals(PlacementConstraints.RACK)) {
minScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(), appId,
te.getTargetValues(), Long::max);
maxScopeCardinality = tm.getRackCardinalityByOp(node.getRackName(), appId,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/11] hadoop git commit: YARN-7669. API and interface modifications
for placement constraint processor. (asuresh)
Posted by as...@apache.org.
YARN-7669. API and interface modifications for placement constraint processor. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d70b0467
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d70b0467
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d70b0467
Branch: refs/heads/YARN-6592
Commit: d70b0467c4e5c5017f81064f71efbb7e7be4f774
Parents: 3967631
Author: Arun Suresh <as...@apache.org>
Authored: Tue Dec 19 22:47:46 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:49:57 2018 -0800
----------------------------------------------------------------------
.../yarn/ams/ApplicationMasterServiceUtils.java | 16 +
.../api/protocolrecords/AllocateResponse.java | 23 +
.../api/records/RejectedSchedulingRequest.java | 70 +++
.../yarn/api/records/RejectionReason.java | 44 ++
.../src/main/proto/yarn_protos.proto | 10 +
.../src/main/proto/yarn_service_protos.proto | 1 +
.../impl/pb/AllocateResponsePBImpl.java | 85 ++++
.../yarn/api/records/impl/pb/ProtoUtils.java | 16 +
.../pb/RejectedSchedulingRequestPBImpl.java | 148 +++++++
.../records/impl/pb/ResourceSizingPBImpl.java | 8 +
.../impl/pb/SchedulingRequestPBImpl.java | 11 +
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +
.../resourcemanager/RMActiveServiceContext.java | 2 +-
.../yarn/server/resourcemanager/RMContext.java | 2 +-
.../server/resourcemanager/RMContextImpl.java | 2 +-
.../server/resourcemanager/ResourceManager.java | 2 +-
.../constraint/AllocationTagsManager.java | 431 -------------------
.../constraint/AllocationTagsNamespaces.java | 31 --
.../InvalidAllocationTagsQueryException.java | 35 --
.../constraint/AllocationTagsManager.java | 431 +++++++++++++++++++
.../constraint/AllocationTagsNamespaces.java | 31 ++
.../InvalidAllocationTagsQueryException.java | 35 ++
.../api/ConstraintPlacementAlgorithm.java | 43 ++
.../api/ConstraintPlacementAlgorithmInput.java | 32 ++
.../api/ConstraintPlacementAlgorithmOutput.java | 58 +++
...traintPlacementAlgorithmOutputCollector.java | 32 ++
.../constraint/api/PlacedSchedulingRequest.java | 79 ++++
.../constraint/api/SchedulingResponse.java | 70 +++
.../scheduler/constraint/api/package-info.java | 28 ++
.../constraint/TestAllocationTagsManager.java | 328 --------------
.../rmcontainer/TestRMContainerImpl.java | 2 +-
.../scheduler/capacity/TestUtils.java | 2 +-
.../constraint/TestAllocationTagsManager.java | 328 ++++++++++++++
.../scheduler/fifo/TestFifoScheduler.java | 2 +-
34 files changed, 1608 insertions(+), 832 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
index 476da8b..8bdfaf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceUtils.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.ams;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
@@ -86,4 +87,19 @@ public final class ApplicationMasterServiceUtils {
}
allocateResponse.setAllocatedContainers(allocatedContainers);
}
+
+ /**
+ * Add rejected Scheduling Requests to {@link AllocateResponse}.
+ * @param allocateResponse Allocate Response.
+ * @param rejectedRequests Rejected SchedulingRequests.
+ */
+ public static void addToRejectedSchedulingRequests(
+ AllocateResponse allocateResponse,
+ List<RejectedSchedulingRequest> rejectedRequests) {
+ if (allocateResponse.getRejectedSchedulingRequests() != null
+ && !allocateResponse.getRejectedSchedulingRequests().isEmpty()) {
+ rejectedRequests.addAll(allocateResponse.getRejectedSchedulingRequests());
+ }
+ allocateResponse.setRejectedSchedulingRequests(rejectedRequests);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index 655c6dc..52c30e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.protocolrecords;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
@@ -410,6 +412,27 @@ public abstract class AllocateResponse {
public abstract void setContainersFromPreviousAttempts(
List<Container> containersFromPreviousAttempt);
+ /**
+ * Get a list of all SchedulingRequests that the RM has rejected between
+ * this allocate call and the previous one.
+ * @return List of RejectedSchedulingRequests.
+ */
+ @Public
+ @Unstable
+ public List<RejectedSchedulingRequest> getRejectedSchedulingRequests() {
+ return Collections.EMPTY_LIST;
+ }
+
+ /**
+ * Add a list of rejected SchedulingRequests to the AllocateResponse.
+ * @param rejectedRequests List of Rejected Scheduling Requests.
+ */
+ @Private
+ @Unstable
+ public void setRejectedSchedulingRequests(
+ List<RejectedSchedulingRequest> rejectedRequests) {
+ }
+
@Private
@Unstable
public static AllocateResponseBuilder newBuilder() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectedSchedulingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectedSchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectedSchedulingRequest.java
new file mode 100644
index 0000000..6e2d95b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectedSchedulingRequest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * This encapsulates a Rejected SchedulingRequest. It contains the offending
+ * Scheduling Request along with the reason for rejection.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public abstract class RejectedSchedulingRequest {
+
+ /**
+ * Create new RejectedSchedulingRequest.
+ * @param reason Rejection Reason.
+ * @param request Rejected Scheduling Request.
+ * @return RejectedSchedulingRequest.
+ */
+ public static RejectedSchedulingRequest newInstance(RejectionReason reason,
+ SchedulingRequest request) {
+ RejectedSchedulingRequest instance =
+ Records.newRecord(RejectedSchedulingRequest.class);
+ instance.setReason(reason);
+ instance.setRequest(request);
+ return instance;
+ }
+
+ /**
+ * Get Rejection Reason.
+ * @return Rejection reason.
+ */
+ public abstract RejectionReason getReason();
+
+ /**
+ * Set Rejection Reason.
+ * @param reason Rejection Reason.
+ */
+ public abstract void setReason(RejectionReason reason);
+
+ /**
+ * Get the Rejected Scheduling Request.
+ * @return SchedulingRequest.
+ */
+ public abstract SchedulingRequest getRequest();
+
+ /**
+ * Set the SchedulingRequest.
+ * @param request SchedulingRequest.
+ */
+ public abstract void setRequest(SchedulingRequest request);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectionReason.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectionReason.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectionReason.java
new file mode 100644
index 0000000..afbc2ed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/RejectionReason.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Reason for rejecting a Scheduling Request.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public enum RejectionReason {
+ /**
+ * This is used to indicate a possible constraint violation. For eg. If the
+ * App requested anti-affinity across 5 container requests, but only 4 nodes
+ * exist. Another eg. could be if tag A has affinity with tag B and tag B has
+ * affinity with tag C, but tag A has anti-affinity with tag C, all at a rack
+ * scope - and only 1 rack exists. Essentially all situations where the
+ * Algorithm cannot assign a Node to SchedulingRequest.
+ */
+ COULD_NOT_PLACE_ON_NODE,
+ /**
+ * This is used to indicate when after the Algorithm has placed a Scheduling
+ * Request at a node, but the commit failed because the Queue has no
+ * capacity etc. This can be a transient situation.
+ */
+ COULD_NOT_SCHEDULE_ON_NODE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index fdc39a7..5cb1177 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -424,6 +424,16 @@ enum AMCommandProto {
AM_SHUTDOWN = 2;
}
+enum RejectionReasonProto {
+ RRP_COULD_NOT_PLACE_ON_NODE = 1;
+ RRP_COULD_NOT_SCHEDULE_ON_NODE = 2;
+}
+
+message RejectedSchedulingRequestProto {
+ required RejectionReasonProto reason = 1;
+ required SchedulingRequestProto request = 2;
+}
+
message PreemptionMessageProto {
optional StrictPreemptionContractProto strictContract = 1;
optional PreemptionContractProto contract = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index e49c4e3..92a65ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -120,6 +120,7 @@ message AllocateResponseProto {
repeated UpdateContainerErrorProto update_errors = 15;
repeated UpdatedContainerProto updated_containers = 16;
repeated ContainerProto containers_from_previous_attempts = 17;
+ repeated RejectedSchedulingRequestProto rejected_scheduling_requests = 18;
}
enum SchedulerResourceTypes {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index 5ca1e73..3ab5563 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
@@ -47,9 +48,11 @@ import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PriorityPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.records.impl.pb.RejectedSchedulingRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.UpdatedContainerPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.hadoop.yarn.proto.YarnProtos.CollectorInfoProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
@@ -81,6 +84,7 @@ public class AllocateResponsePBImpl extends AllocateResponse {
private List<NodeReport> updatedNodes = null;
private List<UpdateContainerError> updateErrors = null;
+ private List<RejectedSchedulingRequest> rejectedRequests = null;
private PreemptionMessage preempt;
private Token amrmToken = null;
private Priority appPriority = null;
@@ -140,6 +144,13 @@ public class AllocateResponsePBImpl extends AllocateResponse {
getContainerStatusProtoIterable(this.completedContainersStatuses);
builder.addAllCompletedContainerStatuses(iterable);
}
+ if (this.rejectedRequests != null) {
+ builder.clearRejectedSchedulingRequests();
+ Iterable<YarnProtos.RejectedSchedulingRequestProto> iterable =
+ getRejectedSchedulingRequestsProtoIterable(
+ this.rejectedRequests);
+ builder.addAllRejectedSchedulingRequests(iterable);
+ }
if (this.updatedNodes != null) {
builder.clearUpdatedNodes();
Iterable<NodeReportProto> iterable =
@@ -471,6 +482,24 @@ public class AllocateResponsePBImpl extends AllocateResponse {
containersFromPreviousAttempts.addAll(containers);
}
+ @Override
+ public synchronized List<RejectedSchedulingRequest>
+ getRejectedSchedulingRequests() {
+ initRejectedRequestsList();
+ return this.rejectedRequests;
+ }
+
+ @Override
+ public synchronized void setRejectedSchedulingRequests(
+ List<RejectedSchedulingRequest> rejectedReqs) {
+ if (rejectedReqs == null) {
+ return;
+ }
+ initRejectedRequestsList();
+ this.rejectedRequests.clear();
+ this.rejectedRequests.addAll(rejectedReqs);
+ }
+
private synchronized void initLocalUpdatedContainerList() {
if (this.updatedContainers != null) {
return;
@@ -528,6 +557,20 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
}
+ private synchronized void initRejectedRequestsList() {
+ if (this.rejectedRequests != null) {
+ return;
+ }
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<YarnProtos.RejectedSchedulingRequestProto> list =
+ p.getRejectedSchedulingRequestsList();
+ rejectedRequests = new ArrayList<>();
+
+ for (YarnProtos.RejectedSchedulingRequestProto c : list) {
+ rejectedRequests.add(convertFromProtoFormat(c));
+ }
+ }
+
private synchronized void initLocalNewNMTokenList() {
if (nmTokens != null) {
return;
@@ -712,6 +755,38 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
};
}
+
+ private synchronized Iterable<YarnProtos.RejectedSchedulingRequestProto>
+ getRejectedSchedulingRequestsProtoIterable(
+ final List<RejectedSchedulingRequest> rejectedReqsList) {
+ maybeInitBuilder();
+ return new Iterable<YarnProtos.RejectedSchedulingRequestProto>() {
+ @Override
+ public Iterator<YarnProtos.RejectedSchedulingRequestProto> iterator() {
+ return new Iterator<YarnProtos.RejectedSchedulingRequestProto>() {
+
+ private Iterator<RejectedSchedulingRequest> iter =
+ rejectedReqsList.iterator();
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public synchronized YarnProtos.RejectedSchedulingRequestProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public synchronized void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+ }
+ };
+ }
private synchronized Iterable<NodeReportProto>
getNodeReportProtoIterable(
@@ -808,6 +883,16 @@ public class AllocateResponsePBImpl extends AllocateResponse {
return ((ContainerStatusPBImpl)t).getProto();
}
+ private synchronized RejectedSchedulingRequestPBImpl convertFromProtoFormat(
+ YarnProtos.RejectedSchedulingRequestProto p) {
+ return new RejectedSchedulingRequestPBImpl(p);
+ }
+
+ private synchronized YarnProtos.RejectedSchedulingRequestProto
+ convertToProtoFormat(RejectedSchedulingRequest t) {
+ return ((RejectedSchedulingRequestPBImpl)t).getProto();
+ }
+
private synchronized ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
return new ResourcePBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 168d864..76e86ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.RejectionReason;
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
@@ -233,6 +234,21 @@ public class ProtoUtils {
}
/*
+ * RejectionReason
+ */
+ private static final String REJECTION_REASON_PREFIX = "RRP_";
+ public static YarnProtos.RejectionReasonProto convertToProtoFormat(
+ RejectionReason e) {
+ return YarnProtos.RejectionReasonProto
+ .valueOf(REJECTION_REASON_PREFIX + e.name());
+ }
+ public static RejectionReason convertFromProtoFormat(
+ YarnProtos.RejectionReasonProto e) {
+ return RejectionReason.valueOf(e.name()
+ .replace(REJECTION_REASON_PREFIX, ""));
+ }
+
+ /*
* ByteBuffer
*/
public static ByteBuffer convertFromProtoFormat(ByteString byteString) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/RejectedSchedulingRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/RejectedSchedulingRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/RejectedSchedulingRequestPBImpl.java
new file mode 100644
index 0000000..ed78551
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/RejectedSchedulingRequestPBImpl.java
@@ -0,0 +1,148 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import com.google.protobuf.TextFormat;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
+import org.apache.hadoop.yarn.api.records.RejectionReason;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.proto.YarnProtos;
+
+/**
+ * Implementation of RejectedSchedulingRequest.
+ */
+public class RejectedSchedulingRequestPBImpl extends RejectedSchedulingRequest {
+
+ private YarnProtos.RejectedSchedulingRequestProto proto =
+ YarnProtos.RejectedSchedulingRequestProto.getDefaultInstance();
+ private YarnProtos.RejectedSchedulingRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+ private SchedulingRequest request;
+
+ public RejectedSchedulingRequestPBImpl() {
+ builder = YarnProtos.RejectedSchedulingRequestProto.newBuilder();
+ }
+
+ public RejectedSchedulingRequestPBImpl(
+ YarnProtos.RejectedSchedulingRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized YarnProtos.RejectedSchedulingRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private synchronized void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private synchronized void mergeLocalToBuilder() {
+ if (this.request != null) {
+ builder.setRequest(convertToProtoFormat(this.request));
+ }
+ }
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = YarnProtos.RejectedSchedulingRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized RejectionReason getReason() {
+ YarnProtos.RejectedSchedulingRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasReason()) {
+ return null;
+ }
+ return ProtoUtils.convertFromProtoFormat(p.getReason());
+ }
+
+ @Override
+ public synchronized void setReason(RejectionReason reason) {
+ maybeInitBuilder();
+ if (reason == null) {
+ builder.clearReason();
+ return;
+ }
+ builder.setReason(ProtoUtils.convertToProtoFormat(reason));
+ }
+
+ @Override
+ public synchronized SchedulingRequest getRequest() {
+ YarnProtos.RejectedSchedulingRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (this.request != null) {
+ return this.request;
+ }
+ if (!p.hasRequest()) {
+ return null;
+ }
+ this.request = convertFromProtoFormat(p.getRequest());
+ return this.request;
+ }
+
+ @Override
+ public synchronized void setRequest(SchedulingRequest req) {
+ maybeInitBuilder();
+ if (null == req) {
+ builder.clearRequest();
+ }
+ this.request = req;
+ }
+
+ private synchronized YarnProtos.SchedulingRequestProto convertToProtoFormat(
+ SchedulingRequest r) {
+ return ((SchedulingRequestPBImpl)r).getProto();
+ }
+
+ private synchronized SchedulingRequestPBImpl convertFromProtoFormat(
+ YarnProtos.SchedulingRequestProto p) {
+ return new SchedulingRequestPBImpl(p);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
index f98e488..4054837 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
@@ -114,4 +114,12 @@ public class ResourceSizingPBImpl extends ResourceSizing {
private ResourceProto convertToProtoFormat(Resource r) {
return ProtoUtils.convertToProtoFormat(r);
}
+
+ @Override
+ public String toString() {
+ return "ResourceSizingPBImpl{" +
+ "numAllocations=" + getNumAllocations() +
+ ", resources=" + getResources() +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
index 305856a..1f86043 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
@@ -279,4 +279,15 @@ public class SchedulingRequestPBImpl extends SchedulingRequest {
}
return false;
}
+
+ @Override
+ public String toString() {
+ return "SchedulingRequestPBImpl{" +
+ "priority=" + getPriority() +
+ ", allocationReqId=" + getAllocationRequestId() +
+ ", executionType=" + getExecutionType() +
+ ", allocationTags=" + getAllocationTags() +
+ ", resourceSizing=" + getResourceSizing() +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index a0b907d..ae80910 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -138,6 +138,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueStatistics;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.RejectedSchedulingRequest;
import org.apache.hadoop.yarn.api.records.ReservationAllocationState;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
import org.apache.hadoop.yarn.api.records.ReservationId;
@@ -436,6 +437,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(ResourceTypeInfo.class);
generateByNewInstance(ResourceSizing.class);
generateByNewInstance(SchedulingRequest.class);
+ generateByNewInstance(RejectedSchedulingRequest.class);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 6ee3a4c..4d0c230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
@@ -43,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetime
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 62899d9..00da108 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
@@ -44,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 315fdc1..da50ef8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
@@ -50,6 +49,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 8d1000e..adda465 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV1Pu
import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV2Publisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
@@ -94,6 +93,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsManager.java
deleted file mode 100644
index b67fab9..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsManager.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * /
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.constraint;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.SchedulingRequest;
-import org.apache.log4j.Logger;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.LongBinaryOperator;
-
-/**
- * Support storing maps between container-tags/applications and
- * nodes. This will be required by affinity/anti-affinity implementation and
- * cardinality.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class AllocationTagsManager {
-
- private static final Logger LOG = Logger.getLogger(
- AllocationTagsManager.class);
-
- private ReentrantReadWriteLock.ReadLock readLock;
- private ReentrantReadWriteLock.WriteLock writeLock;
-
- // Application's tags to node
- private Map<ApplicationId, NodeToCountedTags> perAppMappings =
- new HashMap<>();
-
- // Global tags to node mapping (used to fast return aggregated tags
- // cardinality across apps)
- private NodeToCountedTags globalMapping = new NodeToCountedTags();
-
- /**
- * Store node to counted tags.
- */
- @VisibleForTesting
- static class NodeToCountedTags {
- // Map<NodeId, Map<Tag, Count>>
- private Map<NodeId, Map<String, Long>> nodeToTagsWithCount =
- new HashMap<>();
-
- // protected by external locks
- private void addTagsToNode(NodeId nodeId, Set<String> tags) {
- Map<String, Long> innerMap = nodeToTagsWithCount.computeIfAbsent(nodeId,
- k -> new HashMap<>());
-
- for (String tag : tags) {
- Long count = innerMap.get(tag);
- if (count == null) {
- innerMap.put(tag, 1L);
- } else{
- innerMap.put(tag, count + 1);
- }
- }
- }
-
- // protected by external locks
- private void addTagToNode(NodeId nodeId, String tag) {
- Map<String, Long> innerMap = nodeToTagsWithCount.computeIfAbsent(nodeId,
- k -> new HashMap<>());
-
- Long count = innerMap.get(tag);
- if (count == null) {
- innerMap.put(tag, 1L);
- } else{
- innerMap.put(tag, count + 1);
- }
- }
-
- private void removeTagFromInnerMap(Map<String, Long> innerMap, String tag) {
- Long count = innerMap.get(tag);
- if (count > 1) {
- innerMap.put(tag, count - 1);
- } else {
- if (count <= 0) {
- LOG.warn(
- "Trying to remove tags from node, however the count already"
- + " becomes 0 or less, it could be a potential bug.");
- }
- innerMap.remove(tag);
- }
- }
-
- private void removeTagsFromNode(NodeId nodeId, Set<String> tags) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
- if (innerMap == null) {
- LOG.warn("Failed to find node=" + nodeId
- + " while trying to remove tags, please double check.");
- return;
- }
-
- for (String tag : tags) {
- removeTagFromInnerMap(innerMap, tag);
- }
-
- if (innerMap.isEmpty()) {
- nodeToTagsWithCount.remove(nodeId);
- }
- }
-
- private void removeTagFromNode(NodeId nodeId, String tag) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
- if (innerMap == null) {
- LOG.warn("Failed to find node=" + nodeId
- + " while trying to remove tags, please double check.");
- return;
- }
-
- removeTagFromInnerMap(innerMap, tag);
-
- if (innerMap.isEmpty()) {
- nodeToTagsWithCount.remove(nodeId);
- }
- }
-
- private long getCardinality(NodeId nodeId, String tag) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
- if (innerMap == null) {
- return 0;
- }
- Long value = innerMap.get(tag);
- return value == null ? 0 : value;
- }
-
- private long getCardinality(NodeId nodeId, Set<String> tags,
- LongBinaryOperator op) {
- Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
- if (innerMap == null) {
- return 0;
- }
-
- long returnValue = 0;
- boolean firstTag = true;
-
- if (tags != null && !tags.isEmpty()) {
- for (String tag : tags) {
- Long value = innerMap.get(tag);
- if (value == null) {
- value = 0L;
- }
-
- if (firstTag) {
- returnValue = value;
- firstTag = false;
- continue;
- }
-
- returnValue = op.applyAsLong(returnValue, value);
- }
- } else {
- // Similar to above if, but only iterate values for better performance
- for (long value : innerMap.values()) {
- // For the first value, we will not apply op
- if (firstTag) {
- returnValue = value;
- firstTag = false;
- continue;
- }
- returnValue = op.applyAsLong(returnValue, value);
- }
- }
- return returnValue;
- }
-
- private boolean isEmpty() {
- return nodeToTagsWithCount.isEmpty();
- }
-
- @VisibleForTesting
- public Map<NodeId, Map<String, Long>> getNodeToTagsWithCount() {
- return nodeToTagsWithCount;
- }
- }
-
- @VisibleForTesting
- Map<ApplicationId, NodeToCountedTags> getPerAppMappings() {
- return perAppMappings;
- }
-
- @VisibleForTesting
- NodeToCountedTags getGlobalMapping() {
- return globalMapping;
- }
-
- public AllocationTagsManager() {
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- readLock = lock.readLock();
- writeLock = lock.writeLock();
- }
-
- /**
- * Notify container allocated on a node.
- *
- * @param nodeId allocated node.
- * @param applicationId applicationId
- * @param containerId container id.
- * @param allocationTags allocation tags, see
- * {@link SchedulingRequest#getAllocationTags()}
- * application_id will be added to allocationTags.
- */
- public void addContainer(NodeId nodeId, ApplicationId applicationId,
- ContainerId containerId, Set<String> allocationTags) {
- String applicationIdTag =
- AllocationTagsNamespaces.APP_ID + applicationId.toString();
-
- boolean useSet = false;
- if (allocationTags != null && !allocationTags.isEmpty()) {
- // Copy before edit it.
- allocationTags = new HashSet<>(allocationTags);
- allocationTags.add(applicationIdTag);
- useSet = true;
- }
-
- writeLock.lock();
- try {
- NodeToCountedTags perAppTagsMapping = perAppMappings.computeIfAbsent(
- applicationId, k -> new NodeToCountedTags());
-
- if (useSet) {
- perAppTagsMapping.addTagsToNode(nodeId, allocationTags);
- globalMapping.addTagsToNode(nodeId, allocationTags);
- } else {
- perAppTagsMapping.addTagToNode(nodeId, applicationIdTag);
- globalMapping.addTagToNode(nodeId, applicationIdTag);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Added container=" + containerId + " with tags=[" + StringUtils
- .join(allocationTags, ",") + "]");
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Notify container removed.
- *
- * @param nodeId nodeId
- * @param applicationId applicationId
- * @param containerId containerId.
- * @param allocationTags allocation tags for given container
- */
- public void removeContainer(NodeId nodeId, ApplicationId applicationId,
- ContainerId containerId, Set<String> allocationTags) {
- String applicationIdTag =
- AllocationTagsNamespaces.APP_ID + applicationId.toString();
- boolean useSet = false;
-
- if (allocationTags != null && !allocationTags.isEmpty()) {
- // Copy before edit it.
- allocationTags = new HashSet<>(allocationTags);
- allocationTags.add(applicationIdTag);
- useSet = true;
- }
-
- writeLock.lock();
- try {
- NodeToCountedTags perAppTagsMapping = perAppMappings.get(applicationId);
- if (perAppTagsMapping == null) {
- return;
- }
-
- if (useSet) {
- perAppTagsMapping.removeTagsFromNode(nodeId, allocationTags);
- globalMapping.removeTagsFromNode(nodeId, allocationTags);
- } else {
- perAppTagsMapping.removeTagFromNode(nodeId, applicationIdTag);
- globalMapping.removeTagFromNode(nodeId, applicationIdTag);
- }
-
- if (perAppTagsMapping.isEmpty()) {
- perAppMappings.remove(applicationId);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Removed container=" + containerId + " with tags=[" + StringUtils
- .join(allocationTags, ",") + "]");
- }
- } finally {
- writeLock.unlock();
- }
- }
-
- /**
- * Get cardinality for following conditions. External can pass-in a binary op
- * to implement customized logic. *
- * @param nodeId nodeId, required.
- * @param applicationId applicationId. When null is specified, return
- * aggregated cardinality among all nodes.
- * @param tag allocation tag, see
- * {@link SchedulingRequest#getAllocationTags()},
- * When multiple tags specified. Returns cardinality
- * depends on op. If a specified tag doesn't exist,
- * 0 will be its cardinality.
- * When null/empty tags specified, all tags
- * (of the node/app) will be considered.
- * @return cardinality of specified query on the node.
- * @throws InvalidAllocationTagsQueryException when illegal query
- * parameter specified
- */
- public long getNodeCardinality(NodeId nodeId, ApplicationId applicationId,
- String tag) throws InvalidAllocationTagsQueryException {
- readLock.lock();
-
- try {
- if (nodeId == null) {
- throw new InvalidAllocationTagsQueryException(
- "Must specify nodeId/tags/op to query cardinality");
- }
-
- NodeToCountedTags mapping;
- if (applicationId != null) {
- mapping = perAppMappings.get(applicationId);
- } else{
- mapping = globalMapping;
- }
-
- if (mapping == null) {
- return 0;
- }
-
- return mapping.getCardinality(nodeId, tag);
- } finally {
- readLock.unlock();
- }
- }
-
- /**
- * Check if given tag exists on node.
- *
- * @param nodeId nodeId, required.
- * @param applicationId applicationId. When null is specified, return
- * aggregated cardinality among all nodes.
- * @param tag allocation tag, see
- * {@link SchedulingRequest#getAllocationTags()},
- * When multiple tags specified. Returns cardinality
- * depends on op. If a specified tag doesn't exist,
- * 0 will be its cardinality.
- * When null/empty tags specified, all tags
- * (of the node/app) will be considered.
- * @return cardinality of specified query on the node.
- * @throws InvalidAllocationTagsQueryException when illegal query
- * parameter specified
- */
- public boolean allocationTagExistsOnNode(NodeId nodeId,
- ApplicationId applicationId, String tag)
- throws InvalidAllocationTagsQueryException {
- return getNodeCardinality(nodeId, applicationId, tag) > 0;
- }
-
- /**
- * Get cardinality for following conditions. External can pass-in a binary op
- * to implement customized logic.
- *
- * @param nodeId nodeId, required.
- * @param applicationId applicationId. When null is specified, return
- * aggregated cardinality among all nodes.
- * @param tags allocation tags, see
- * {@link SchedulingRequest#getAllocationTags()},
- * When multiple tags specified. Returns cardinality
- * depends on op. If a specified tag doesn't exist, 0
- * will be its cardinality. When null/empty tags
- * specified, all tags (of the node/app) will be
- * considered.
- * @param op operator. Such as Long::max, Long::sum, etc. Required.
- * This sparameter only take effect when #values >= 2.
- * @return cardinality of specified query on the node.
- * @throws InvalidAllocationTagsQueryException when illegal query
- * parameter specified
- */
- public long getNodeCardinalityByOp(NodeId nodeId, ApplicationId applicationId,
- Set<String> tags, LongBinaryOperator op)
- throws InvalidAllocationTagsQueryException {
- readLock.lock();
-
- try {
- if (nodeId == null || op == null) {
- throw new InvalidAllocationTagsQueryException(
- "Must specify nodeId/tags/op to query cardinality");
- }
-
- NodeToCountedTags mapping;
- if (applicationId != null) {
- mapping = perAppMappings.get(applicationId);
- } else{
- mapping = globalMapping;
- }
-
- if (mapping == null) {
- return 0;
- }
-
- return mapping.getCardinality(nodeId, tags, op);
- } finally {
- readLock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsNamespaces.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsNamespaces.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsNamespaces.java
deleted file mode 100644
index 893ff1c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/AllocationTagsNamespaces.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * /
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.constraint;
-
-/**
- * Predefined namespaces for tags
- *
- * Same as namespace of resource types. Namespaces of placement tags are start
- * with alphabets and ended with "/"
- */
-public class AllocationTagsNamespaces {
- public static final String APP_ID = "yarn_app_id/";
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/InvalidAllocationTagsQueryException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/InvalidAllocationTagsQueryException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/InvalidAllocationTagsQueryException.java
deleted file mode 100644
index 5519e39..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/constraint/InvalidAllocationTagsQueryException.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- * /
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.constraint;
-
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-/**
- * Exception when invalid parameter specified to do placement tags related
- * queries.
- */
-public class InvalidAllocationTagsQueryException extends YarnException {
- private static final long serialVersionUID = 12312831974894L;
-
- public InvalidAllocationTagsQueryException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
new file mode 100644
index 0000000..c278606
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsManager.java
@@ -0,0 +1,431 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.LongBinaryOperator;
+
+/**
+ * Support storing maps between container-tags/applications and
+ * nodes. This will be required by affinity/anti-affinity implementation and
+ * cardinality.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class AllocationTagsManager {
+
+ private static final Logger LOG = Logger.getLogger(
+ AllocationTagsManager.class);
+
+ private ReentrantReadWriteLock.ReadLock readLock;
+ private ReentrantReadWriteLock.WriteLock writeLock;
+
+ // Application's tags to node
+ private Map<ApplicationId, NodeToCountedTags> perAppMappings =
+ new HashMap<>();
+
+ // Global tags to node mapping (used to fast return aggregated tags
+ // cardinality across apps)
+ private NodeToCountedTags globalMapping = new NodeToCountedTags();
+
+ /**
+ * Store node to counted tags.
+ */
+ @VisibleForTesting
+ static class NodeToCountedTags {
+ // Map<NodeId, Map<Tag, Count>>
+ private Map<NodeId, Map<String, Long>> nodeToTagsWithCount =
+ new HashMap<>();
+
+ // protected by external locks
+ private void addTagsToNode(NodeId nodeId, Set<String> tags) {
+ Map<String, Long> innerMap = nodeToTagsWithCount.computeIfAbsent(nodeId,
+ k -> new HashMap<>());
+
+ for (String tag : tags) {
+ Long count = innerMap.get(tag);
+ if (count == null) {
+ innerMap.put(tag, 1L);
+ } else{
+ innerMap.put(tag, count + 1);
+ }
+ }
+ }
+
+ // protected by external locks
+ private void addTagToNode(NodeId nodeId, String tag) {
+ Map<String, Long> innerMap = nodeToTagsWithCount.computeIfAbsent(nodeId,
+ k -> new HashMap<>());
+
+ Long count = innerMap.get(tag);
+ if (count == null) {
+ innerMap.put(tag, 1L);
+ } else{
+ innerMap.put(tag, count + 1);
+ }
+ }
+
+ private void removeTagFromInnerMap(Map<String, Long> innerMap, String tag) {
+ Long count = innerMap.get(tag);
+ if (count > 1) {
+ innerMap.put(tag, count - 1);
+ } else {
+ if (count <= 0) {
+ LOG.warn(
+ "Trying to remove tags from node, however the count already"
+ + " becomes 0 or less, it could be a potential bug.");
+ }
+ innerMap.remove(tag);
+ }
+ }
+
+ private void removeTagsFromNode(NodeId nodeId, Set<String> tags) {
+ Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ if (innerMap == null) {
+ LOG.warn("Failed to find node=" + nodeId
+ + " while trying to remove tags, please double check.");
+ return;
+ }
+
+ for (String tag : tags) {
+ removeTagFromInnerMap(innerMap, tag);
+ }
+
+ if (innerMap.isEmpty()) {
+ nodeToTagsWithCount.remove(nodeId);
+ }
+ }
+
+ private void removeTagFromNode(NodeId nodeId, String tag) {
+ Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ if (innerMap == null) {
+ LOG.warn("Failed to find node=" + nodeId
+ + " while trying to remove tags, please double check.");
+ return;
+ }
+
+ removeTagFromInnerMap(innerMap, tag);
+
+ if (innerMap.isEmpty()) {
+ nodeToTagsWithCount.remove(nodeId);
+ }
+ }
+
+ private long getCardinality(NodeId nodeId, String tag) {
+ Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ if (innerMap == null) {
+ return 0;
+ }
+ Long value = innerMap.get(tag);
+ return value == null ? 0 : value;
+ }
+
+ private long getCardinality(NodeId nodeId, Set<String> tags,
+ LongBinaryOperator op) {
+ Map<String, Long> innerMap = nodeToTagsWithCount.get(nodeId);
+ if (innerMap == null) {
+ return 0;
+ }
+
+ long returnValue = 0;
+ boolean firstTag = true;
+
+ if (tags != null && !tags.isEmpty()) {
+ for (String tag : tags) {
+ Long value = innerMap.get(tag);
+ if (value == null) {
+ value = 0L;
+ }
+
+ if (firstTag) {
+ returnValue = value;
+ firstTag = false;
+ continue;
+ }
+
+ returnValue = op.applyAsLong(returnValue, value);
+ }
+ } else {
+ // Similar to above if, but only iterate values for better performance
+ for (long value : innerMap.values()) {
+ // For the first value, we will not apply op
+ if (firstTag) {
+ returnValue = value;
+ firstTag = false;
+ continue;
+ }
+ returnValue = op.applyAsLong(returnValue, value);
+ }
+ }
+ return returnValue;
+ }
+
+ private boolean isEmpty() {
+ return nodeToTagsWithCount.isEmpty();
+ }
+
+ @VisibleForTesting
+ public Map<NodeId, Map<String, Long>> getNodeToTagsWithCount() {
+ return nodeToTagsWithCount;
+ }
+ }
+
+ @VisibleForTesting
+ Map<ApplicationId, NodeToCountedTags> getPerAppMappings() {
+ return perAppMappings;
+ }
+
+ @VisibleForTesting
+ NodeToCountedTags getGlobalMapping() {
+ return globalMapping;
+ }
+
+ public AllocationTagsManager() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ }
+
+ /**
+ * Notify container allocated on a node.
+ *
+ * @param nodeId allocated node.
+ * @param applicationId applicationId
+ * @param containerId container id.
+ * @param allocationTags allocation tags, see
+ * {@link SchedulingRequest#getAllocationTags()}
+ * application_id will be added to allocationTags.
+ */
+ public void addContainer(NodeId nodeId, ApplicationId applicationId,
+ ContainerId containerId, Set<String> allocationTags) {
+ String applicationIdTag =
+ AllocationTagsNamespaces.APP_ID + applicationId.toString();
+
+ boolean useSet = false;
+ if (allocationTags != null && !allocationTags.isEmpty()) {
+ // Copy before edit it.
+ allocationTags = new HashSet<>(allocationTags);
+ allocationTags.add(applicationIdTag);
+ useSet = true;
+ }
+
+ writeLock.lock();
+ try {
+ NodeToCountedTags perAppTagsMapping = perAppMappings.computeIfAbsent(
+ applicationId, k -> new NodeToCountedTags());
+
+ if (useSet) {
+ perAppTagsMapping.addTagsToNode(nodeId, allocationTags);
+ globalMapping.addTagsToNode(nodeId, allocationTags);
+ } else {
+ perAppTagsMapping.addTagToNode(nodeId, applicationIdTag);
+ globalMapping.addTagToNode(nodeId, applicationIdTag);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Added container=" + containerId + " with tags=[" + StringUtils
+ .join(allocationTags, ",") + "]");
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Notify container removed.
+ *
+ * @param nodeId nodeId
+ * @param applicationId applicationId
+ * @param containerId containerId.
+ * @param allocationTags allocation tags for given container
+ */
+ public void removeContainer(NodeId nodeId, ApplicationId applicationId,
+ ContainerId containerId, Set<String> allocationTags) {
+ String applicationIdTag =
+ AllocationTagsNamespaces.APP_ID + applicationId.toString();
+ boolean useSet = false;
+
+ if (allocationTags != null && !allocationTags.isEmpty()) {
+ // Copy before edit it.
+ allocationTags = new HashSet<>(allocationTags);
+ allocationTags.add(applicationIdTag);
+ useSet = true;
+ }
+
+ writeLock.lock();
+ try {
+ NodeToCountedTags perAppTagsMapping = perAppMappings.get(applicationId);
+ if (perAppTagsMapping == null) {
+ return;
+ }
+
+ if (useSet) {
+ perAppTagsMapping.removeTagsFromNode(nodeId, allocationTags);
+ globalMapping.removeTagsFromNode(nodeId, allocationTags);
+ } else {
+ perAppTagsMapping.removeTagFromNode(nodeId, applicationIdTag);
+ globalMapping.removeTagFromNode(nodeId, applicationIdTag);
+ }
+
+ if (perAppTagsMapping.isEmpty()) {
+ perAppMappings.remove(applicationId);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Removed container=" + containerId + " with tags=[" + StringUtils
+ .join(allocationTags, ",") + "]");
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Get cardinality for following conditions. External can pass-in a binary op
+ * to implement customized logic. *
+ * @param nodeId nodeId, required.
+ * @param applicationId applicationId. When null is specified, return
+ * aggregated cardinality among all nodes.
+ * @param tag allocation tag, see
+ * {@link SchedulingRequest#getAllocationTags()},
+ * When multiple tags specified. Returns cardinality
+ * depends on op. If a specified tag doesn't exist,
+ * 0 will be its cardinality.
+ * When null/empty tags specified, all tags
+ * (of the node/app) will be considered.
+ * @return cardinality of specified query on the node.
+ * @throws InvalidAllocationTagsQueryException when illegal query
+ * parameter specified
+ */
+ public long getNodeCardinality(NodeId nodeId, ApplicationId applicationId,
+ String tag) throws InvalidAllocationTagsQueryException {
+ readLock.lock();
+
+ try {
+ if (nodeId == null) {
+ throw new InvalidAllocationTagsQueryException(
+ "Must specify nodeId/tags/op to query cardinality");
+ }
+
+ NodeToCountedTags mapping;
+ if (applicationId != null) {
+ mapping = perAppMappings.get(applicationId);
+ } else{
+ mapping = globalMapping;
+ }
+
+ if (mapping == null) {
+ return 0;
+ }
+
+ return mapping.getCardinality(nodeId, tag);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ /**
+ * Check if given tag exists on node.
+ *
+ * @param nodeId nodeId, required.
+ * @param applicationId applicationId. When null is specified, return
+ * aggregated cardinality among all nodes.
+ * @param tag allocation tag, see
+ * {@link SchedulingRequest#getAllocationTags()},
+ * When multiple tags specified. Returns cardinality
+ * depends on op. If a specified tag doesn't exist,
+ * 0 will be its cardinality.
+ * When null/empty tags specified, all tags
+ * (of the node/app) will be considered.
+ * @return cardinality of specified query on the node.
+ * @throws InvalidAllocationTagsQueryException when illegal query
+ * parameter specified
+ */
+ public boolean allocationTagExistsOnNode(NodeId nodeId,
+ ApplicationId applicationId, String tag)
+ throws InvalidAllocationTagsQueryException {
+ return getNodeCardinality(nodeId, applicationId, tag) > 0;
+ }
+
+ /**
+ * Get cardinality for following conditions. External can pass-in a binary op
+ * to implement customized logic.
+ *
+ * @param nodeId nodeId, required.
+ * @param applicationId applicationId. When null is specified, return
+ * aggregated cardinality among all nodes.
+ * @param tags allocation tags, see
+ * {@link SchedulingRequest#getAllocationTags()},
+ * When multiple tags specified. Returns cardinality
+ * depends on op. If a specified tag doesn't exist, 0
+ * will be its cardinality. When null/empty tags
+ * specified, all tags (of the node/app) will be
+ * considered.
+ * @param op operator. Such as Long::max, Long::sum, etc. Required.
+ * This sparameter only take effect when #values >= 2.
+ * @return cardinality of specified query on the node.
+ * @throws InvalidAllocationTagsQueryException when illegal query
+ * parameter specified
+ */
+ public long getNodeCardinalityByOp(NodeId nodeId, ApplicationId applicationId,
+ Set<String> tags, LongBinaryOperator op)
+ throws InvalidAllocationTagsQueryException {
+ readLock.lock();
+
+ try {
+ if (nodeId == null || op == null) {
+ throw new InvalidAllocationTagsQueryException(
+ "Must specify nodeId/tags/op to query cardinality");
+ }
+
+ NodeToCountedTags mapping;
+ if (applicationId != null) {
+ mapping = perAppMappings.get(applicationId);
+ } else{
+ mapping = globalMapping;
+ }
+
+ if (mapping == null) {
+ return 0;
+ }
+
+ return mapping.getCardinality(nodeId, tags, op);
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsNamespaces.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsNamespaces.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsNamespaces.java
new file mode 100644
index 0000000..43fcfe5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/AllocationTagsNamespaces.java
@@ -0,0 +1,31 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+/**
+ * Predefined namespaces for tags
+ *
+ * Same as namespace of resource types. Namespaces of placement tags are start
+ * with alphabets and ended with "/"
+ */
+public class AllocationTagsNamespaces {
+ public static final String APP_ID = "yarn_app_id/";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/InvalidAllocationTagsQueryException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/InvalidAllocationTagsQueryException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/InvalidAllocationTagsQueryException.java
new file mode 100644
index 0000000..29483a2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/InvalidAllocationTagsQueryException.java
@@ -0,0 +1,35 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+/**
+ * Exception when invalid parameter specified to do placement tags related
+ * queries.
+ */
+public class InvalidAllocationTagsQueryException extends YarnException {
+ private static final long serialVersionUID = 12312831974894L;
+
+ public InvalidAllocationTagsQueryException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithm.java
new file mode 100644
index 0000000..2651663
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithm.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+
+/**
+ * Marker interface for a Constraint Placement. The only contract is that it
+ * should be initialized with the RMContext.
+ */
+public interface ConstraintPlacementAlgorithm {
+
+ /**
+ * Initialize the Algorithm.
+ * @param rmContext RMContext.
+ */
+ void init(RMContext rmContext);
+
+ /**
+ * The Algorithm is expected to compute the placement of the provided
+ * ConstraintPlacementAlgorithmInput and use the collector to aggregate
+ * any output.
+ * @param algorithmInput Input to the Algorithm.
+ * @param collector Collector for output of algorithm.
+ */
+ void place(ConstraintPlacementAlgorithmInput algorithmInput,
+ ConstraintPlacementAlgorithmOutputCollector collector);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d70b0467/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmInput.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmInput.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmInput.java
new file mode 100644
index 0000000..74572b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/api/ConstraintPlacementAlgorithmInput.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.api;
+
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+
+import java.util.Collection;
+
+/**
+ * This encapsulates an input to the Constraint Placement Algorithm. At the
+ * very least it must consist of a collection of SchedulerRequests.
+ */
+public interface ConstraintPlacementAlgorithmInput {
+
+ Collection<SchedulingRequest> getSchedulingRequests();
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/11] hadoop git commit: YARN-7681. Double-check placement
constraints in scheduling phase before actual allocation is made. (Weiwei
Yang via asuresh)
Posted by as...@apache.org.
YARN-7681. Double-check placement constraints in scheduling phase before actual allocation is made. (Weiwei Yang via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f572351
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f572351
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f572351
Branch: refs/heads/YARN-6592
Commit: 3f572351daa5256789c1542691c09e0082906fa6
Parents: 96186f5
Author: Arun Suresh <as...@apache.org>
Authored: Wed Jan 10 09:04:30 2018 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:52:24 2018 -0800
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 23 ++++++++++++++++++++
1 file changed, 23 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f572351/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index f03d7d1..956d840 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -123,6 +123,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.SchedulerContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.InvalidAllocationTagsQueryException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintsUtil;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -2512,6 +2514,27 @@ public class CapacityScheduler extends
ResourceCommitRequest<FiCaSchedulerApp, FiCaSchedulerNode>
resourceCommitRequest = createResourceCommitRequest(
appAttempt, schedulingRequest, schedulerNode);
+
+ // Validate placement constraint is satisfied before
+ // committing the request.
+ try {
+ if (!PlacementConstraintsUtil.canSatisfyConstraints(
+ appAttempt.getApplicationId(),
+ schedulingRequest.getAllocationTags(),
+ schedulerNode,
+ rmContext.getPlacementConstraintManager(),
+ rmContext.getAllocationTagsManager())) {
+ LOG.debug("Failed to allocate container for application "
+ + appAttempt.getApplicationId() + " on node "
+ + schedulerNode.getNodeName()
+ + " because this allocation violates the"
+ + " placement constraint.");
+ return false;
+ }
+ } catch (InvalidAllocationTagsQueryException e) {
+ LOG.warn("Unable to allocate container", e);
+ return false;
+ }
return tryCommit(getClusterResource(), resourceCommitRequest, false);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/11] hadoop git commit: YARN-6596. Introduce Placement Constraint
Manager module. (Konstantinos Karanasos via asuresh)
Posted by as...@apache.org.
YARN-6596. Introduce Placement Constraint Manager module. (Konstantinos Karanasos via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f650d39b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f650d39b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f650d39b
Branch: refs/heads/YARN-6592
Commit: f650d39b8a6890b55dd961781cb751e3a7c38c51
Parents: b3dd520
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 22 13:26:30 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Jan 17 13:50:38 2018 -0800
----------------------------------------------------------------------
.../resourcemanager/RMActiveServiceContext.java | 15 +
.../yarn/server/resourcemanager/RMContext.java | 6 +
.../server/resourcemanager/RMContextImpl.java | 13 +
.../server/resourcemanager/ResourceManager.java | 13 +
.../MemoryPlacementConstraintManager.java | 282 +++++++++++++++++++
.../constraint/PlacementConstraintManager.java | 151 ++++++++++
.../PlacementConstraintManagerService.java | 93 ++++++
.../scheduler/constraint/package-info.java | 29 ++
.../TestPlacementConstraintManagerService.java | 182 ++++++++++++
9 files changed, 784 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 4d0c230..06a1d00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
@@ -109,6 +110,7 @@ public class RMActiveServiceContext {
private RMAppLifetimeMonitor rmAppLifetimeMonitor;
private QueueLimitCalculator queueLimitCalculator;
private AllocationTagsManager allocationTagsManager;
+ private PlacementConstraintManager placementConstraintManager;
public RMActiveServiceContext() {
queuePlacementManager = new PlacementManager();
@@ -413,6 +415,19 @@ public class RMActiveServiceContext {
@Private
@Unstable
+ public PlacementConstraintManager getPlacementConstraintManager() {
+ return placementConstraintManager;
+ }
+
+ @Private
+ @Unstable
+ public void setPlacementConstraintManager(
+ PlacementConstraintManager placementConstraintManager) {
+ this.placementConstraintManager = placementConstraintManager;
+ }
+
+ @Private
+ @Unstable
public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
return rmDelegatedNodeLabelsUpdater;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 00da108..eb91a31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
@@ -171,4 +172,9 @@ public interface RMContext extends ApplicationMasterServiceContext {
AllocationTagsManager getAllocationTagsManager();
void setAllocationTagsManager(AllocationTagsManager allocationTagsManager);
+
+ PlacementConstraintManager getPlacementConstraintManager();
+
+ void setPlacementConstraintManager(
+ PlacementConstraintManager placementConstraintManager);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index da50ef8..0b6be72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.distributed.QueueLimitCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
@@ -516,6 +517,18 @@ public class RMContextImpl implements RMContext {
}
@Override
+ public PlacementConstraintManager getPlacementConstraintManager() {
+ return activeServiceContext.getPlacementConstraintManager();
+ }
+
+ @Override
+ public void setPlacementConstraintManager(
+ PlacementConstraintManager placementConstraintManager) {
+ activeServiceContext
+ .setPlacementConstraintManager(placementConstraintManager);
+ }
+
+ @Override
public RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater() {
return activeServiceContext.getRMDelegatedNodeLabelsUpdater();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index d71f224..2396b94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -94,6 +94,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.MemoryPlacementConstraintManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManagerService;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
@@ -495,6 +497,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected AllocationTagsManager createAllocationTagsManager() {
return new AllocationTagsManager(this.rmContext);
}
+
+ protected PlacementConstraintManagerService
+ createPlacementConstraintManager() {
+ // Use the in memory Placement Constraint Manager.
+ return new MemoryPlacementConstraintManager();
+ }
protected DelegationTokenRenewer createDelegationTokenRenewer() {
return new DelegationTokenRenewer();
@@ -618,6 +626,11 @@ public class ResourceManager extends CompositeService implements Recoverable {
createAllocationTagsManager();
rmContext.setAllocationTagsManager(allocationTagsManager);
+ PlacementConstraintManagerService placementConstraintManager =
+ createPlacementConstraintManager();
+ addService(placementConstraintManager);
+ rmContext.setPlacementConstraintManager(placementConstraintManager);
+
RMDelegatedNodeLabelsUpdater delegatedNodeLabelsUpdater =
createRMDelegatedNodeLabelsUpdater();
if (delegatedNodeLabelsUpdater != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
new file mode 100644
index 0000000..ceff6f6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
@@ -0,0 +1,282 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * In memory implementation of the {@link PlacementConstraintManagerService}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class MemoryPlacementConstraintManager
+ extends PlacementConstraintManagerService {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MemoryPlacementConstraintManager.class);
+
+ private ReentrantReadWriteLock.ReadLock readLock;
+ private ReentrantReadWriteLock.WriteLock writeLock;
+
+ /**
+ * Stores the global constraints that will be manipulated by the cluster
+ * admin. The key of each entry is the tag that will enable the corresponding
+ * constraint.
+ */
+ private Map<String, PlacementConstraint> globalConstraints;
+ /**
+ * Stores the constraints for each application, along with the allocation tags
+ * that will enable each of the constraints for a given application.
+ */
+ private Map<ApplicationId, Map<String, PlacementConstraint>> appConstraints;
+
+ public MemoryPlacementConstraintManager() {
+ this.globalConstraints = new HashMap<>();
+ this.appConstraints = new HashMap<>();
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public void registerApplication(ApplicationId appId,
+ Map<Set<String>, PlacementConstraint> constraintMap) {
+ // Check if app already exists. If not, prepare its constraint map.
+ Map<String, PlacementConstraint> constraintsForApp = new HashMap<>();
+ try {
+ readLock.lock();
+ if (appConstraints.get(appId) != null) {
+ LOG.warn("Application {} has already been registered.", appId);
+ return;
+ }
+ // Go over each sourceTag-constraint pair, validate it, and add it to the
+ // constraint map for this app.
+ for (Map.Entry<Set<String>, PlacementConstraint> entry : constraintMap
+ .entrySet()) {
+ Set<String> sourceTags = entry.getKey();
+ PlacementConstraint constraint = entry.getValue();
+ if (validateConstraint(sourceTags, constraint)) {
+ String sourceTag = getValidSourceTag(sourceTags);
+ constraintsForApp.put(sourceTag, constraint);
+ }
+ }
+ } finally {
+ readLock.unlock();
+ }
+
+ if (constraintsForApp.isEmpty()) {
+ LOG.info("Application {} was registered, but no constraints were added.",
+ appId);
+ }
+ // Update appConstraints.
+ try {
+ writeLock.lock();
+ appConstraints.put(appId, constraintsForApp);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void addConstraint(ApplicationId appId, Set<String> sourceTags,
+ PlacementConstraint placementConstraint, boolean replace) {
+ try {
+ writeLock.lock();
+ Map<String, PlacementConstraint> constraintsForApp =
+ appConstraints.get(appId);
+ if (constraintsForApp == null) {
+ LOG.info("Cannot add constraint to application {}, as it has not "
+ + "been registered yet.", appId);
+ return;
+ }
+
+ addConstraintToMap(constraintsForApp, sourceTags, placementConstraint,
+ replace);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void addGlobalConstraint(Set<String> sourceTags,
+ PlacementConstraint placementConstraint, boolean replace) {
+ try {
+ writeLock.lock();
+ addConstraintToMap(globalConstraints, sourceTags, placementConstraint,
+ replace);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Helper method that adds a constraint to a map for a given source tag.
+ * Assumes there is already a lock on the constraint map.
+ *
+ * @param constraintMap constraint map to which the constraint will be added
+ * @param sourceTags the source tags that will enable this constraint
+ * @param placementConstraint the new constraint to be added
+ * @param replace if true, an existing constraint for these sourceTags will be
+ * replaced with the new one
+ */
+ private void addConstraintToMap(
+ Map<String, PlacementConstraint> constraintMap, Set<String> sourceTags,
+ PlacementConstraint placementConstraint, boolean replace) {
+ if (validateConstraint(sourceTags, placementConstraint)) {
+ String sourceTag = getValidSourceTag(sourceTags);
+ if (constraintMap.get(sourceTag) == null || replace) {
+ if (replace) {
+ LOG.info("Replacing the constraint associated with tag {} with {}.",
+ sourceTag, placementConstraint);
+ }
+ constraintMap.put(sourceTag, placementConstraint);
+ } else {
+ LOG.info("Constraint {} will not be added. There is already a "
+ + "constraint associated with tag {}.",
+ placementConstraint, sourceTag);
+ }
+ }
+ }
+
+ @Override
+ public Map<Set<String>, PlacementConstraint> getConstraints(
+ ApplicationId appId) {
+ try {
+ readLock.lock();
+ if (appConstraints.get(appId) == null) {
+ LOG.info("Application {} is not registered in the Placement "
+ + "Constraint Manager.", appId);
+ return null;
+ }
+
+ // Copy to a new map and return an unmodifiable version of it.
+ // Each key of the map is a set with a single source tag.
+ Map<Set<String>, PlacementConstraint> constraintMap =
+ appConstraints.get(appId).entrySet().stream()
+ .collect(Collectors.toMap(
+ e -> Stream.of(e.getKey()).collect(Collectors.toSet()),
+ e -> e.getValue()));
+
+ return Collections.unmodifiableMap(constraintMap);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public PlacementConstraint getConstraint(ApplicationId appId,
+ Set<String> sourceTags) {
+ if (!validateSourceTags(sourceTags)) {
+ return null;
+ }
+ String sourceTag = getValidSourceTag(sourceTags);
+ try {
+ readLock.lock();
+ if (appConstraints.get(appId) == null) {
+ LOG.info("Application {} is not registered in the Placement "
+ + "Constraint Manager.", appId);
+ return null;
+ }
+ // TODO: Merge this constraint with the global one for this tag, if one
+ // exists.
+ return appConstraints.get(appId).get(sourceTag);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public PlacementConstraint getGlobalConstraint(Set<String> sourceTags) {
+ if (!validateSourceTags(sourceTags)) {
+ return null;
+ }
+ String sourceTag = getValidSourceTag(sourceTags);
+ try {
+ readLock.lock();
+ return globalConstraints.get(sourceTag);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void unregisterApplication(ApplicationId appId) {
+ try {
+ writeLock.lock();
+ appConstraints.remove(appId);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void removeGlobalConstraint(Set<String> sourceTags) {
+ if (!validateSourceTags(sourceTags)) {
+ return;
+ }
+ String sourceTag = getValidSourceTag(sourceTags);
+ try {
+ writeLock.lock();
+ globalConstraints.remove(sourceTag);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public int getNumRegisteredApplications() {
+ try {
+ readLock.lock();
+ return appConstraints.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public int getNumGlobalConstraints() {
+ try {
+ readLock.lock();
+ return globalConstraints.size();
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManager.java
new file mode 100644
index 0000000..7725d0d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManager.java
@@ -0,0 +1,151 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+
+/**
+ * Interface for storing and retrieving placement constraints (see
+ * {@link PlacementConstraint}).
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public interface PlacementConstraintManager {
+
+ /**
+ * Register all placement constraints of an application.
+ *
+ * @param appId the application ID
+ * @param constraintMap the map of allocation tags to constraints for this
+ * application
+ */
+ void registerApplication(ApplicationId appId,
+ Map<Set<String>, PlacementConstraint> constraintMap);
+
+ /**
+ * Add a placement constraint for a given application and a given set of
+ * (source) allocation tags. The constraint will be used on Scheduling
+ * Requests that carry this set of allocation tags.
+ * TODO: Support merge and not only replace when adding a constraint.
+ *
+ * @param appId the application ID
+ * @param sourceTags the set of allocation tags that will enable this
+ * constraint
+ * @param placementConstraint the constraint
+ * @param replace if true, an existing constraint for these tags will be
+ * replaced by the given one
+ */
+ void addConstraint(ApplicationId appId, Set<String> sourceTags,
+ PlacementConstraint placementConstraint, boolean replace);
+
+ /**
+ * Add a placement constraint that will be used globally. These constraints
+ * are added by the cluster administrator.
+ * TODO: Support merge and not only replace when adding a constraint.
+ *
+ * @param sourceTags the allocation tags that will enable this constraint
+ * @param placementConstraint the constraint
+ * @param replace if true, an existing constraint for these tags will be
+ * replaced by the given one
+ */
+ void addGlobalConstraint(Set<String> sourceTags,
+ PlacementConstraint placementConstraint, boolean replace);
+
+ /**
+ * Retrieve all constraints for a given application, along with the allocation
+ * tags that enable each constraint.
+ *
+ * @param appId the application ID
+ * @return the constraints for this application with the associated tags
+ */
+ Map<Set<String>, PlacementConstraint> getConstraints(ApplicationId appId);
+
+ /**
+ * Retrieve the placement constraint that is associated with a set of
+ * allocation tags for a given application.
+ *
+ * @param appId the application ID
+ * @param sourceTags the allocation tags that enable this constraint
+ * @return the constraint
+ */
+ PlacementConstraint getConstraint(ApplicationId appId,
+ Set<String> sourceTags);
+
+ /**
+ * Retrieve a global constraint that is associated with a given set of
+ * allocation tags.
+ *
+ * @param sourceTags the allocation tags that enable this constraint
+ * @return the constraint
+ */
+ PlacementConstraint getGlobalConstraint(Set<String> sourceTags);
+
+ /**
+ * Remove the constraints that correspond to a given application.
+ *
+ * @param appId the application that will be removed.
+ */
+ void unregisterApplication(ApplicationId appId);
+
+ /**
+ * Remove a global constraint that is associated with the given allocation
+ * tags.
+ *
+ * @param sourceTags the allocation tags
+ */
+ void removeGlobalConstraint(Set<String> sourceTags);
+
+ /**
+ * Returns the number of currently registered applications in the Placement
+ * Constraint Manager.
+ *
+ * @return number of registered applications.
+ */
+ int getNumRegisteredApplications();
+
+ /**
+ * Returns the number of global constraints registered in the Placement
+ * Constraint Manager.
+ *
+ * @return number of global constraints.
+ */
+ int getNumGlobalConstraints();
+
+ /**
+ * Validate a placement constraint and the set of allocation tags that will
+ * enable it.
+ *
+ * @param sourceTags the associated allocation tags
+ * @param placementConstraint the constraint
+ * @return true if constraint and tags are valid
+ */
+ default boolean validateConstraint(Set<String> sourceTags,
+ PlacementConstraint placementConstraint) {
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManagerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManagerService.java
new file mode 100644
index 0000000..967f251
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/PlacementConstraintManagerService.java
@@ -0,0 +1,93 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+
+/**
+ * The service that implements the {@link PlacementConstraintManager} interface.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class PlacementConstraintManagerService extends AbstractService
+ implements PlacementConstraintManager {
+
+ protected static final Log LOG =
+ LogFactory.getLog(PlacementConstraintManagerService.class);
+
+ private PlacementConstraintManager placementConstraintManager = null;
+
+ public PlacementConstraintManagerService() {
+ super(PlacementConstraintManagerService.class.getName());
+ }
+
+ @Override
+ public boolean validateConstraint(Set<String> sourceTags,
+ PlacementConstraint placementConstraint) {
+ if (!validateSourceTags(sourceTags)) {
+ return false;
+ }
+ // TODO: Perform actual validation of the constraint (in YARN-6621).
+ // TODO: Perform satisfiability check for constraint.
+ return true;
+ }
+
+ /**
+ * Validates whether the allocation tags that will enable a constraint have
+ * the expected format. At the moment we support a single allocation tag per
+ * constraint.
+ *
+ * @param sourceTags the source allocation tags
+ * @return true if the tags have the expected format
+ */
+ protected boolean validateSourceTags(Set<String> sourceTags) {
+ if (sourceTags.isEmpty()) {
+ LOG.warn("A placement constraint cannot be associated with an empty "
+ + "set of tags.");
+ return false;
+ }
+ if (sourceTags.size() > 1) {
+ LOG.warn("Only a single tag can be associated with a placement "
+ + "constraint currently.");
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * This method will return a single allocation tag. It should be called after
+ * validating the tags by calling {@link #validateSourceTags}.
+ *
+ * @param sourceTags the source allocation tags
+ * @return the single source tag
+ */
+ protected String getValidSourceTag(Set<String> sourceTags) {
+ return sourceTags.iterator().next();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/package-info.java
new file mode 100644
index 0000000..cbb7a55
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement
+ * contains classes related to scheduling containers using placement
+ * constraints.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f650d39b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintManagerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintManagerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintManagerService.java
new file mode 100644
index 0000000..abcab1a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/TestPlacementConstraintManagerService.java
@@ -0,0 +1,182 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute;
+
+import java.util.AbstractMap.SimpleEntry;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link PlacementConstraintManagerService}.
+ */
+public class TestPlacementConstraintManagerService {
+
+ private PlacementConstraintManagerService pcm;
+
+ protected PlacementConstraintManagerService createPCM() {
+ return new MemoryPlacementConstraintManager();
+ }
+
+ private ApplicationId appId1, appId2;
+ private PlacementConstraint c1, c2, c3, c4;
+ private Set<String> sourceTag1, sourceTag2, sourceTag3, sourceTag4;
+ private Map<Set<String>, PlacementConstraint> constraintMap1, constraintMap2;
+
+ @Before
+ public void before() {
+ this.pcm = createPCM();
+
+ // Build appIDs, constraints, source tags, and constraint map.
+ long ts = System.currentTimeMillis();
+ appId1 = BuilderUtils.newApplicationId(ts, 123);
+ appId2 = BuilderUtils.newApplicationId(ts, 234);
+
+ c1 = PlacementConstraints.build(targetIn(NODE, allocationTag("hbase-m")));
+ c2 = PlacementConstraints.build(targetIn(RACK, allocationTag("hbase-rs")));
+ c3 = PlacementConstraints
+ .build(targetNotIn(NODE, nodeAttribute("java", "1.8")));
+ c4 = PlacementConstraints
+ .build(targetCardinality(RACK, 2, 10, allocationTag("zk")));
+
+ sourceTag1 = new HashSet<>(Arrays.asList("spark"));
+ sourceTag2 = new HashSet<>(Arrays.asList("zk"));
+ sourceTag3 = new HashSet<>(Arrays.asList("storm"));
+ sourceTag4 = new HashSet<>(Arrays.asList("hbase-m", "hbase-sec"));
+
+ constraintMap1 = Stream
+ .of(new SimpleEntry<>(sourceTag1, c1),
+ new SimpleEntry<>(sourceTag2, c2))
+ .collect(Collectors.toMap(SimpleEntry::getKey, SimpleEntry::getValue));
+
+ constraintMap2 = Stream.of(new SimpleEntry<>(sourceTag3, c4))
+ .collect(Collectors.toMap(SimpleEntry::getKey, SimpleEntry::getValue));
+ }
+
+ @Test
+ public void testRegisterUnregisterApps() {
+ Assert.assertEquals(0, pcm.getNumRegisteredApplications());
+
+ // Register two applications.
+ pcm.registerApplication(appId1, constraintMap1);
+ Assert.assertEquals(1, pcm.getNumRegisteredApplications());
+ Map<Set<String>, PlacementConstraint> constrMap =
+ pcm.getConstraints(appId1);
+ Assert.assertNotNull(constrMap);
+ Assert.assertEquals(2, constrMap.size());
+ Assert.assertNotNull(constrMap.get(sourceTag1));
+ Assert.assertNotNull(constrMap.get(sourceTag2));
+
+ pcm.registerApplication(appId2, constraintMap2);
+ Assert.assertEquals(2, pcm.getNumRegisteredApplications());
+ constrMap = pcm.getConstraints(appId2);
+ Assert.assertNotNull(constrMap);
+ Assert.assertEquals(1, constrMap.size());
+ Assert.assertNotNull(constrMap.get(sourceTag3));
+ Assert.assertNull(constrMap.get(sourceTag2));
+
+ // Try to register the same app again.
+ pcm.registerApplication(appId2, constraintMap1);
+ Assert.assertEquals(2, pcm.getNumRegisteredApplications());
+
+ // Unregister appId1.
+ pcm.unregisterApplication(appId1);
+ Assert.assertEquals(1, pcm.getNumRegisteredApplications());
+ Assert.assertNull(pcm.getConstraints(appId1));
+ Assert.assertNotNull(pcm.getConstraints(appId2));
+ }
+
+ @Test
+ public void testAddConstraint() {
+ // Cannot add constraint to unregistered app.
+ Assert.assertEquals(0, pcm.getNumRegisteredApplications());
+ pcm.addConstraint(appId1, sourceTag1, c1, false);
+ Assert.assertEquals(0, pcm.getNumRegisteredApplications());
+
+ // Register application.
+ pcm.registerApplication(appId1, new HashMap<>());
+ Assert.assertEquals(1, pcm.getNumRegisteredApplications());
+ Assert.assertEquals(0, pcm.getConstraints(appId1).size());
+
+ // Add two constraints.
+ pcm.addConstraint(appId1, sourceTag1, c1, false);
+ pcm.addConstraint(appId1, sourceTag2, c3, false);
+ Assert.assertEquals(2, pcm.getConstraints(appId1).size());
+
+ // Constraint for sourceTag1 should not be replaced.
+ pcm.addConstraint(appId1, sourceTag1, c2, false);
+ Assert.assertEquals(2, pcm.getConstraints(appId1).size());
+ Assert.assertEquals(c1, pcm.getConstraint(appId1, sourceTag1));
+ Assert.assertNotEquals(c2, pcm.getConstraint(appId1, sourceTag1));
+
+ // Now c2 should replace c1 for sourceTag1.
+ pcm.addConstraint(appId1, sourceTag1, c2, true);
+ Assert.assertEquals(2, pcm.getConstraints(appId1).size());
+ Assert.assertEquals(c2, pcm.getConstraint(appId1, sourceTag1));
+ }
+
+ @Test
+ public void testGlobalConstraints() {
+ Assert.assertEquals(0, pcm.getNumGlobalConstraints());
+ pcm.addGlobalConstraint(sourceTag1, c1, false);
+ Assert.assertEquals(1, pcm.getNumGlobalConstraints());
+ Assert.assertNotNull(pcm.getGlobalConstraint(sourceTag1));
+
+ // Constraint for sourceTag1 should not be replaced.
+ pcm.addGlobalConstraint(sourceTag1, c2, false);
+ Assert.assertEquals(1, pcm.getNumGlobalConstraints());
+ Assert.assertEquals(c1, pcm.getGlobalConstraint(sourceTag1));
+ Assert.assertNotEquals(c2, pcm.getGlobalConstraint(sourceTag1));
+
+ // Now c2 should replace c1 for sourceTag1.
+ pcm.addGlobalConstraint(sourceTag1, c2, true);
+ Assert.assertEquals(1, pcm.getNumGlobalConstraints());
+ Assert.assertEquals(c2, pcm.getGlobalConstraint(sourceTag1));
+
+ pcm.removeGlobalConstraint(sourceTag1);
+ Assert.assertEquals(0, pcm.getNumGlobalConstraints());
+ }
+
+ @Test
+ public void testValidateConstraint() {
+ // At the moment we only disallow multiple source tags to be associated with
+ // a constraint. TODO: More tests to be added for YARN-6621.
+ Assert.assertTrue(pcm.validateConstraint(sourceTag1, c1));
+ Assert.assertFalse(pcm.validateConstraint(sourceTag4, c1));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org