You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by zj...@apache.org on 2014/04/15 22:37:44 UTC
svn commit: r1587717 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemana...
Author: zjshen
Date: Tue Apr 15 20:37:44 2014
New Revision: 1587717
URL: http://svn.apache.org/r1587717
Log:
YARN-1892. Improved some logs in the scheduler. Contributed by Jian He.
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1587717&r1=1587716&r2=1587717&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Tue Apr 15 20:37:44 2014
@@ -73,6 +73,8 @@ Release 2.4.1 - UNRELEASED
IMPROVEMENTS
+ YARN-1892. Improved some logs in the scheduler. (Jian He via zjshen)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1587717&r1=1587716&r2=1587717&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Tue Apr 15 20:37:44 2014
@@ -343,10 +343,6 @@ public class NodeStatusUpdaterImpl exten
+ ", " + nodeHealthStatus.getHealthReport());
}
List<ContainerStatus> containersStatuses = getContainerStatuses();
- if (LOG.isDebugEnabled()) {
- LOG.debug(this.nodeId + " sending out status for "
- + containersStatuses.size() + " containers");
- }
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
createKeepAliveApplicationList(), nodeHealthStatus);
@@ -373,7 +369,8 @@ public class NodeStatusUpdaterImpl exten
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Sending out container statuses: " + containerStatuses);
+ LOG.debug("Sending out " + containerStatuses.size()
+ + " container statuses: " + containerStatuses);
}
return containerStatuses;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java?rev=1587717&r1=1587716&r2=1587717&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java Tue Apr 15 20:37:44 2014
@@ -255,13 +255,15 @@ public class SchedulerApplicationAttempt
this.reservedContainers.put(priority, reservedContainers);
}
reservedContainers.put(node.getNodeID(), rmContainer);
-
- LOG.info("Application " + getApplicationId()
- + " reserved container " + rmContainer
- + " on node " + node + ", currently has " + reservedContainers.size()
- + " at priority " + priority
- + "; currentReservation " + currentReservation.getMemory());
-
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Application attempt " + getApplicationAttemptId()
+ + " reserved container " + rmContainer + " on node " + node
+ + ". This attempt currently has " + reservedContainers.size()
+ + " reserved containers at priority " + priority
+ + "; currentReservation " + currentReservation.getMemory());
+ }
+
return rmContainer;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1587717&r1=1587716&r2=1587717&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Apr 15 20:37:44 2014
@@ -107,8 +107,6 @@ public class LeafQueue implements CSQueu
private final Resource maximumAllocation;
private final float minimumAllocationFactor;
- private RMContainerTokenSecretManager containerTokenSecretManager;
-
private Map<String, User> users = new HashMap<String, User>();
private final QueueMetrics metrics;
@@ -149,7 +147,6 @@ public class LeafQueue implements CSQueu
Resources.ratio(resourceCalculator,
Resources.subtract(maximumAllocation, minimumAllocation),
maximumAllocation);
- this.containerTokenSecretManager = cs.getContainerTokenSecretManager();
float capacity =
(float)cs.getConfiguration().getCapacity(getQueuePath()) / 100;
@@ -560,7 +557,7 @@ public class LeafQueue implements CSQueu
return queueName + ": " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
- "usedResources=" + usedResources +
+ "usedResources=" + usedResources + ", " +
"usedCapacity=" + getUsedCapacity() + ", " +
"absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
@@ -949,15 +946,16 @@ public class LeafQueue implements CSQueu
Resources.add(usedResources, required),
clusterResource);
if (potentialNewCapacity > absoluteMaxCapacity) {
- LOG.info(getQueueName() +
- " usedResources: " + usedResources +
- " clusterResources: " + clusterResource +
- " currentCapacity " +
- Resources.divide(resourceCalculator, clusterResource,
- usedResources, clusterResource) +
- " required " + required +
- " potentialNewCapacity: " + potentialNewCapacity + " ( " +
- " max-capacity: " + absoluteMaxCapacity + ")");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getQueueName()
+ + " usedResources: " + usedResources
+ + " clusterResources: " + clusterResource
+ + " currentCapacity "
+ + Resources.divide(resourceCalculator, clusterResource,
+ usedResources, clusterResource) + " required " + required
+ + " potentialNewCapacity: " + potentialNewCapacity + " ( "
+ + " max-capacity: " + absoluteMaxCapacity + ")");
+ }
return false;
}
return true;
@@ -1300,7 +1298,7 @@ public class LeafQueue implements CSQueu
ResourceRequest request, NodeType type, RMContainer rmContainer) {
if (LOG.isDebugEnabled()) {
LOG.debug("assignContainers: node=" + node.getNodeName()
- + " application=" + application.getApplicationId().getId()
+ + " application=" + application.getApplicationId()
+ " priority=" + priority.getPriority()
+ " request=" + request + " type=" + type);
}
@@ -1352,14 +1350,10 @@ public class LeafQueue implements CSQueu
allocatedContainer);
LOG.info("assignedContainer" +
- " application=" + application.getApplicationId() +
+ " application attempt=" + application.getApplicationAttemptId() +
" container=" + container +
- " containerId=" + container.getId() +
" queue=" + this +
- " usedCapacity=" + getUsedCapacity() +
- " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
- " cluster=" + clusterResource);
+ " clusterResource=" + clusterResource);
return container.getResource();
} else {
@@ -1367,13 +1361,11 @@ public class LeafQueue implements CSQueu
reserve(application, priority, node, rmContainer, container);
LOG.info("Reserved container " +
- " application=" + application.getApplicationId() +
+ " application attempt=" + application.getApplicationAttemptId() +
" resource=" + request.getCapability() +
" queue=" + this.toString() +
- " usedCapacity=" + getUsedCapacity() +
- " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
- " cluster=" + clusterResource);
+ " node=" + node +
+ " clusterResource=" + clusterResource);
return request.getCapability();
}
@@ -1440,11 +1432,7 @@ public class LeafQueue implements CSQueu
application, container.getResource());
LOG.info("completedContainer" +
" container=" + container +
- " resource=" + container.getResource() +
" queue=" + this +
- " usedCapacity=" + getUsedCapacity() +
- " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
" cluster=" + clusterResource);
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java?rev=1587717&r1=1587716&r2=1587717&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java Tue Apr 15 20:37:44 2014
@@ -231,12 +231,19 @@ public class FiCaSchedulerNode extends S
" on node " + this);
}
- LOG.info("Updated reserved container " +
- reservedContainer.getContainer().getId() + " on node " +
- this + " for application " + application);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updated reserved container "
+ + reservedContainer.getContainer().getId() + " on node " + this
+ + " for application attempt "
+ + application.getApplicationAttemptId());
+ }
} else {
- LOG.info("Reserved container " + reservedContainer.getContainer().getId() +
- " on node " + this + " for application " + application);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Reserved container "
+ + reservedContainer.getContainer().getId() + " on node " + this
+ + " for application attempt "
+ + application.getApplicationAttemptId());
+ }
}
this.reservedContainer = reservedContainer;
}