You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/07/03 16:59:36 UTC
hadoop git commit: YARN-8459. Improve Capacity Scheduler logs to
debug invalid states. Contributed by Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/trunk 344f32471 -> 51654a396
YARN-8459. Improve Capacity Scheduler logs to debug invalid states. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51654a39
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51654a39
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51654a39
Branch: refs/heads/trunk
Commit: 51654a3962bcd0482c0e1dd32765e9533e3f9158
Parents: 344f324
Author: Sunil G <su...@apache.org>
Authored: Tue Jul 3 09:59:20 2018 -0700
Committer: Sunil G <su...@apache.org>
Committed: Tue Jul 3 09:59:20 2018 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 39 +++++++++++++-------
.../scheduler/capacity/ParentQueue.java | 36 ++++++++++++------
.../allocator/AbstractContainerAllocator.java | 13 ++++---
.../scheduler/common/fica/FiCaSchedulerApp.java | 5 +++
4 files changed, 63 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51654a39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index b59636a..37f56de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1234,8 +1234,10 @@ public class CapacityScheduler extends
updateDemandForQueue.getOrderingPolicy().demandUpdated(application);
}
- LOG.info("Allocation for application " + applicationAttemptId + " : " +
- allocation + " with cluster resource : " + getClusterResource());
+ if (LOG.isDebugEnabled()) {
+ LOG.info("Allocation for application " + applicationAttemptId + " : "
+ + allocation + " with cluster resource : " + getClusterResource());
+ }
return allocation;
}
@@ -1479,10 +1481,17 @@ public class CapacityScheduler extends
private CSAssignment allocateContainerOnSingleNode(
CandidateNodeSet<FiCaSchedulerNode> candidates, FiCaSchedulerNode node,
boolean withNodeHeartbeat) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Trying to schedule on node: " + node.getNodeName() + ", available: "
+ + node.getUnallocatedResource());
+ }
+
// Backward compatible way to make sure previous behavior which allocation
// driven by node heartbeat works.
if (getNode(node.getNodeID()) != node) {
- LOG.error("Trying to schedule on a removed node, please double check.");
+ LOG.error("Trying to schedule on a removed node, please double check, "
+ + "nodeId=" + node.getNodeID());
return null;
}
@@ -1496,14 +1505,19 @@ public class CapacityScheduler extends
FiCaSchedulerApp reservedApplication = getCurrentAttemptForContainer(
reservedContainer.getContainerId());
if (reservedApplication == null) {
- LOG.error("Trying to schedule for a finished app, please double check.");
+ LOG.error(
+ "Trying to schedule for a finished app, please double check. nodeId="
+ + node.getNodeID() + " container=" + reservedContainer
+ .getContainerId());
return null;
}
// Try to fulfill the reservation
- LOG.info(
- "Trying to fulfill reservation for application " + reservedApplication
- .getApplicationId() + " on node: " + node.getNodeID());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to fulfill reservation for application "
+ + reservedApplication.getApplicationId() + " on node: " + node
+ .getNodeID());
+ }
LeafQueue queue = ((LeafQueue) reservedApplication.getQueue());
assignment = queue.assignContainers(getClusterResource(), candidates,
@@ -1567,12 +1581,6 @@ public class CapacityScheduler extends
return null;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug(
- "Trying to schedule on node: " + node.getNodeName() + ", available: "
- + node.getUnallocatedResource());
- }
-
return allocateOrReserveNewContainers(candidates, withNodeHeartbeat);
}
@@ -2888,6 +2896,11 @@ public class CapacityScheduler extends
LOG.info("Failed to accept allocation proposal");
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Allocation proposal accepted=" + isSuccess + ", proposal="
+ + request);
+ }
+
// Update unconfirmed allocated resource.
if (updateUnconfirmedAllocatedResource) {
app.decUnconfirmedRes(request.getTotalAllocatedResource());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51654a39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index bb4823e..2363b88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -90,6 +90,8 @@ public class ParentQueue extends AbstractCSQueue {
private QueueOrderingPolicy queueOrderingPolicy;
+ private long lastSkipQueueDebugLoggingTimestamp = -1;
+
public ParentQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
@@ -539,9 +541,14 @@ public class ParentQueue extends AbstractCSQueue {
if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
&& !accessibleToPartition(candidates.getPartition())) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Skip this queue=" + getQueuePath()
- + ", because it is not able to access partition=" + candidates
- .getPartition());
+ long now = System.currentTimeMillis();
+ // Do logging every 1 sec to avoid excessive logging.
+ if (now - this.lastSkipQueueDebugLoggingTimestamp > 1000) {
+ LOG.debug("Skip this queue=" + getQueuePath()
+ + ", because it is not able to access partition=" + candidates
+ .getPartition());
+ this.lastSkipQueueDebugLoggingTimestamp = now;
+ }
}
ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
@@ -561,10 +568,15 @@ public class ParentQueue extends AbstractCSQueue {
if (!super.hasPendingResourceRequest(candidates.getPartition(),
clusterResource, schedulingMode)) {
if (LOG.isDebugEnabled()) {
- LOG.debug("Skip this queue=" + getQueuePath()
- + ", because it doesn't need more resource, schedulingMode="
- + schedulingMode.name() + " node-partition=" + candidates
- .getPartition());
+ long now = System.currentTimeMillis();
+ // Do logging every 1 sec to avoid excessive logging.
+ if (now - this.lastSkipQueueDebugLoggingTimestamp > 1000) {
+ LOG.debug("Skip this queue=" + getQueuePath()
+ + ", because it doesn't need more resource, schedulingMode="
+ + schedulingMode.name() + " node-partition=" + candidates
+ .getPartition());
+ this.lastSkipQueueDebugLoggingTimestamp = now;
+ }
}
ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
@@ -666,12 +678,12 @@ public class ParentQueue extends AbstractCSQueue {
assignment.setIncreasedAllocation(
assignedToChild.isIncreasedAllocation());
- LOG.info("assignedContainer" + " queue=" + getQueueName()
- + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
- + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed()
- + " cluster=" + clusterResource);
-
if (LOG.isDebugEnabled()) {
+ LOG.debug("assignedContainer reserved=" + isReserved + " queue="
+ + getQueueName() + " usedCapacity=" + getUsedCapacity()
+ + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
+ + queueUsage.getUsed() + " cluster=" + clusterResource);
+
LOG.debug(
"ParentQ=" + getQueueName() + " assignedSoFarInThisIteration="
+ assignment.getResource() + " usedCapacity="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51654a39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
index 36665d6..bfc129f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/AbstractContainerAllocator.java
@@ -93,11 +93,14 @@ public abstract class AbstractContainerAllocator {
assignment.setType(result.getContainerNodeType());
if (result.getAllocationState() == AllocationState.RESERVED) {
- // This is a reserved container
- LOG.info("Reserved container " + " application="
- + application.getApplicationId() + " resource=" + allocatedResource
- + " queue=" + appInfo.getQueueName()
- + " cluster=" + clusterResource);
+ if (LOG.isDebugEnabled()) {
+ // This is a reserved container
+ // Since re-reservation could happen again and again for already
+ // reserved containers. only do this in debug log.
+ LOG.debug("Reserved container " + " application=" + application
+ .getApplicationId() + " resource=" + allocatedResource + " queue="
+ + appInfo.getQueueName() + " cluster=" + clusterResource);
+ }
assignment.getAssignmentInformation().addReservationDetails(
updatedContainer, application.getCSLeafQueue().getQueuePath());
assignment.getAssignmentInformation().incrReservations();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51654a39/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 3ec8191..3b1b82c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -607,6 +607,11 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
schedulerContainer.getRmContainer(),
schedulerContainer.getRmContainer().getContainer(),
reReservation);
+
+ LOG.info("Reserved container=" + rmContainer.getContainerId()
+ + ", on node=" + schedulerContainer.getSchedulerNode()
+ + " with resource=" + rmContainer
+ .getAllocatedOrReservedResource());
}
}
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org