You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/08/26 00:44:37 UTC
[25/50] [abbrv] hadoop git commit: YARN-8683. Support to display
pending scheduling requests in RM app attempt page. Contributed by Tao Yang.
YARN-8683. Support to display pending scheduling requests in RM app attempt page. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54d0bf89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54d0bf89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54d0bf89
Branch: refs/heads/HDFS-12943
Commit: 54d0bf8935e35aad0f4d67df358ceb970cfcd713
Parents: d3fef7a
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue Aug 21 19:00:31 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue Aug 21 19:00:31 2018 +0800
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 10 ++++
.../scheduler/AppSchedulingInfo.java | 17 +++++++
.../scheduler/YarnScheduler.java | 10 ++++
.../placement/AppPlacementAllocator.java | 6 +++
.../LocalityAppPlacementAllocator.java | 5 ++
.../SingleConstraintAppPlacementAllocator.java | 4 +-
.../webapp/RMAppAttemptBlock.java | 24 +++++++--
.../resourcemanager/webapp/dao/AppInfo.java | 10 ++++
.../webapp/dao/ResourceRequestInfo.java | 52 ++++++++++++++++++--
9 files changed, 129 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index d2e81a5..9d2b058 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -878,6 +878,16 @@ public abstract class AbstractYarnScheduler
}
@Override
+ public List<SchedulingRequest> getPendingSchedulingRequestsForAttempt(
+ ApplicationAttemptId attemptId) {
+ SchedulerApplicationAttempt attempt = getApplicationAttempt(attemptId);
+ if (attempt != null) {
+ return attempt.getAppSchedulingInfo().getAllSchedulingRequests();
+ }
+ return null;
+ }
+
+ @Override
public Priority checkAndGetApplicationPriority(
Priority priorityRequestedByApp, UserGroupInformation user,
String queueName, ApplicationId applicationId) throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 8074f06..d63d2b82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -476,6 +476,23 @@ public class AppSchedulingInfo {
return ret;
}
+ /**
+ * Fetch SchedulingRequests.
+ * @return All pending SchedulingRequests.
+ */
+ public List<SchedulingRequest> getAllSchedulingRequests() {
+ List<SchedulingRequest> ret = new ArrayList<>();
+ try {
+ this.readLock.lock();
+ schedulerKeyToAppPlacementAllocator.values().stream()
+ .filter(ap -> ap.getSchedulingRequest() != null)
+ .forEach(ap -> ret.add(ap.getSchedulingRequest()));
+ } finally {
+ this.readLock.unlock();
+ }
+ return ret;
+ }
+
public PendingAsk getNextPendingAsk() {
try {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 43d55c4..0f7a5b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -364,6 +364,16 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
ApplicationAttemptId attemptId);
/**
+ * Get pending scheduling request for specified application attempt.
+ *
+ * @param attemptId the id of the application attempt
+ *
+ * @return pending scheduling requests
+ */
+ List<SchedulingRequest> getPendingSchedulingRequestsForAttempt(
+ ApplicationAttemptId attemptId);
+
+ /**
* Get cluster max priority.
*
* @return maximum priority of cluster
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
index df58157..088b3dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/AppPlacementAllocator.java
@@ -192,4 +192,10 @@ public abstract class AppPlacementAllocator<N extends SchedulerNode> {
this.rmContext = rmContext;
this.schedulerRequestKey = schedulerRequestKey;
}
+
+ /**
+ * Get pending SchedulingRequest.
+ * @return SchedulingRequest
+ */
+ public abstract SchedulingRequest getSchedulingRequest();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
index e1239a9..f1df343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -425,4 +425,9 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
writeLock.unlock();
}
}
+
+ @Override
+ public SchedulingRequest getSchedulingRequest() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
index 2b610f2..914f35d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/SingleConstraintAppPlacementAllocator.java
@@ -505,8 +505,8 @@ public class SingleConstraintAppPlacementAllocator<N extends SchedulerNode>
}
}
- @VisibleForTesting
- SchedulingRequest getSchedulingRequest() {
+ @Override
+ public SchedulingRequest getSchedulingRequest() {
return schedulingRequest;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 43a6ac9..99a569a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -87,10 +87,14 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
.h3("Total Outstanding Resource Requests: "
+ getTotalResource(resourceRequests))
.table("#resourceRequests").thead().tr().th(".priority", "Priority")
+ .th(".allocationRequestId", "AllocationRequestId")
.th(".resource", "ResourceName").th(".capacity", "Capability")
.th(".containers", "NumContainers")
.th(".relaxlocality", "RelaxLocality")
- .th(".labelexpression", "NodeLabelExpression").__().__().tbody();
+ .th(".labelexpression", "NodeLabelExpression")
+ .th(".executiontype", "ExecutionType")
+ .th(".allocationTags", "AllocationTags")
+ .th(".placementConstraint", "PlacementConstraint").__().__().tbody();
StringBuilder resourceRequestTableData = new StringBuilder("[\n");
for (ResourceRequestInfo resourceRequest : resourceRequests) {
@@ -99,7 +103,11 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
}
resourceRequestTableData.append("[\"")
.append(String.valueOf(resourceRequest.getPriority())).append("\",\"")
- .append(resourceRequest.getResourceName()).append("\",\"")
+ .append(String.valueOf(resourceRequest.getAllocationRequestId()))
+ .append("\",\"")
+ .append(resourceRequest.getResourceName() == null ? "N/A"
+ : resourceRequest.getResourceName())
+ .append("\",\"")
.append(StringEscapeUtils.escapeEcmaScript(StringEscapeUtils
.escapeHtml4(String.valueOf(resourceRequest.getCapability()))))
.append("\",\"")
@@ -109,6 +117,15 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
.append("\",\"")
.append(resourceRequest.getNodeLabelExpression() == null ? "N/A"
: resourceRequest.getNodeLabelExpression())
+ .append("\",\"")
+ .append(resourceRequest.getExecutionTypeRequest() == null ? "N/A"
+ : resourceRequest.getExecutionTypeRequest().getExecutionType())
+ .append("\",\"")
+ .append(resourceRequest.getAllocationTags() == null ? "N/A" :
+ StringUtils.join(resourceRequest.getAllocationTags(), ","))
+ .append("\",\"")
+ .append(resourceRequest.getPlacementConstraint() == null ? "N/A"
+ : resourceRequest.getPlacementConstraint())
.append("\"],\n");
}
if (resourceRequestTableData
@@ -132,7 +149,8 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
if (request.getNumContainers() == 0) {
continue;
}
- if (request.getResourceName().equals(ResourceRequest.ANY)) {
+ if (request.getResourceName() == null || request.getResourceName()
+ .equals(ResourceRequest.ANY)) {
Resources.addTo(
totalResource,
Resources.multiply(request.getCapability().getResource(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 9d82bc7..63b6fe0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -230,6 +231,15 @@ public class AppInfo {
resourceRequests.add(new ResourceRequestInfo(req));
}
}
+
+ List<SchedulingRequest> schedulingRequestsRaw = rm.getRMContext()
+ .getScheduler().getPendingSchedulingRequestsForAttempt(
+ attempt.getAppAttemptId());
+ if (schedulingRequestsRaw != null) {
+ for (SchedulingRequest req : schedulingRequestsRaw) {
+ resourceRequests.add(new ResourceRequestInfo(req));
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54d0bf89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java
index 030af45..beab9d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceRequestInfo.java
@@ -21,11 +21,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
+import java.util.Set;
/**
* Simple class representing a resource request.
@@ -36,6 +38,8 @@ public class ResourceRequestInfo {
@XmlElement(name = "priority")
private int priority;
+ @XmlElement(name = "allocationRequestId")
+ private long allocationRequestId;
@XmlElement(name = "resourceName")
private String resourceName;
@XmlElement(name = "capability")
@@ -50,16 +54,17 @@ public class ResourceRequestInfo {
@XmlElement(name = "executionTypeRequest")
private ExecutionTypeRequestInfo executionTypeRequest;
- @XmlElement(name = "executionType")
- private String executionType;
- @XmlElement(name = "enforceExecutionType")
- private boolean enforceExecutionType;
+ @XmlElement(name = "placementConstraint")
+ private String placementConstraint;
+ @XmlElement(name = "allocationTags")
+ private Set<String> allocationTags;
public ResourceRequestInfo() {
}
public ResourceRequestInfo(ResourceRequest request) {
priority = request.getPriority().getPriority();
+ allocationRequestId = request.getAllocationRequestId();
resourceName = request.getResourceName();
capability = new ResourceInfo(request.getCapability());
numContainers = request.getNumContainers();
@@ -71,6 +76,21 @@ public class ResourceRequestInfo {
}
}
+ public ResourceRequestInfo(SchedulingRequest request) {
+ priority = request.getPriority().getPriority();
+ allocationRequestId = request.getAllocationRequestId();
+ capability = new ResourceInfo(request.getResourceSizing().getResources());
+ numContainers = request.getResourceSizing().getNumAllocations();
+ if (request.getExecutionType() != null) {
+ executionTypeRequest =
+ new ExecutionTypeRequestInfo(request.getExecutionType());
+ }
+ allocationTags = request.getAllocationTags();
+ if (request.getPlacementConstraint() != null) {
+ placementConstraint = request.getPlacementConstraint().toString();
+ }
+ }
+
public Priority getPriority() {
return Priority.newInstance(priority);
}
@@ -128,4 +148,28 @@ public class ResourceRequestInfo {
public ExecutionTypeRequestInfo getExecutionTypeRequest() {
return executionTypeRequest;
}
+
+ public String getPlacementConstraint() {
+ return placementConstraint;
+ }
+
+ public void setPlacementConstraint(String placementConstraint) {
+ this.placementConstraint = placementConstraint;
+ }
+
+ public Set<String> getAllocationTags() {
+ return allocationTags;
+ }
+
+ public void setAllocationTags(Set<String> allocationTags) {
+ this.allocationTags = allocationTags;
+ }
+
+ public long getAllocationRequestId() {
+ return allocationRequestId;
+ }
+
+ public void setAllocationRequestId(long allocationRequestId) {
+ this.allocationRequestId = allocationRequestId;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org