You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/28 22:27:51 UTC
[24/24] hadoop git commit: YARN-8809. Refactor AbstractYarnScheduler
and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo
Chen via asuresh)
YARN-8809. Refactor AbstractYarnScheduler and CapacityScheduler OPPORTUNISTIC container completion codepaths. (Haibo Chen via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d217ee3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d217ee3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d217ee3
Branch: refs/heads/YARN-1011
Commit: 8d217ee3c38c18bc1929cfd981a7296898d4a202
Parents: 0a54014
Author: Arun Suresh <as...@apache.org>
Authored: Fri Sep 21 12:02:00 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 28 14:15:03 2018 -0700
----------------------------------------------------------------------
.../scheduler/AbstractYarnScheduler.java | 32 ++++++++------------
.../scheduler/capacity/CapacityScheduler.java | 25 ++++++++++++++-
.../scheduler/fair/FairScheduler.java | 11 ++++---
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 25 +++++++++++++++
5 files changed, 70 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a8f5093..63f534f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -677,25 +677,12 @@ public abstract class AbstractYarnScheduler
}
if (rmContainer.getExecutionType() == ExecutionType.GUARANTEED) {
- completedContainerInternal(rmContainer, containerStatus, event);
+ completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
completeOustandingUpdatesWhichAreReserved(
rmContainer, containerStatus, event);
} else {
- ContainerId containerId = rmContainer.getContainerId();
- // Inform the container
- rmContainer.handle(
- new RMContainerFinishedEvent(containerId, containerStatus, event));
- SchedulerApplicationAttempt schedulerAttempt =
- getCurrentAttemptForContainer(containerId);
- if (schedulerAttempt != null) {
- schedulerAttempt.removeRMContainer(containerId);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Completed container: " + rmContainer.getContainerId() +
- " in state: " + rmContainer.getState() + " event:" + event);
- }
- getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
- rmContainer.getContainerId(), false);
+ completeOpportunisticContainerInternal(rmContainer, containerStatus,
+ event);
}
// If the container is getting killed in ACQUIRED state, the requester (AM
@@ -705,6 +692,12 @@ public abstract class AbstractYarnScheduler
recoverResourceRequestForContainer(rmContainer);
}
+ protected void completeOpportunisticContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ completeGuaranteedContainerInternal(rmContainer, containerStatus, event);
+ }
+
// Optimization:
// Check if there are in-flight container updates and complete the
// associated temp containers. These are removed when the app completes,
@@ -722,7 +715,7 @@ public abstract class AbstractYarnScheduler
.getReservedSchedulerKey().getContainerToUpdate();
if (containerToUpdate != null &&
containerToUpdate.equals(containerStatus.getContainerId())) {
- completedContainerInternal(resContainer,
+ completeGuaranteedContainerInternal(resContainer,
ContainerStatus.newInstance(resContainer.getContainerId(),
containerStatus.getState(), containerStatus
.getDiagnostics(),
@@ -732,8 +725,9 @@ public abstract class AbstractYarnScheduler
}
}
- // clean up a completed container
- protected abstract void completedContainerInternal(RMContainer rmContainer,
+ // clean up a completed guaranteed container
+ protected abstract void completeGuaranteedContainerInternal(
+ RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event);
protected void releaseContainers(List<ContainerId> containers,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 0b7115b..c06122f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -2066,7 +2067,29 @@ public class CapacityScheduler extends
}
@Override
- protected void completedContainerInternal(
+ protected void completeOpportunisticContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ ContainerId containerId = rmContainer.getContainerId();
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerFinishedEvent(containerId, containerStatus, event));
+ SchedulerApplicationAttempt schedulerAttempt =
+ getCurrentAttemptForContainer(containerId);
+ if (schedulerAttempt != null) {
+ schedulerAttempt.removeRMContainer(containerId);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Completed OPPORTUNISTIC container: " +
+ rmContainer.getContainerId() + " in state: " +
+ rmContainer.getState() + " event:" + event);
+ }
+ getSchedulerNode(rmContainer.getNodeId()).releaseContainer(
+ rmContainer.getContainerId(), false);
+ }
+
+ @Override
+ protected void completeGuaranteedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
Container container = rmContainer.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 31f1961..744776a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -719,11 +719,14 @@ public class FairScheduler extends
}
}
- /**
- * Clean up a completed container.
- */
@Override
- protected void completedContainerInternal(
+ protected void completeGuaranteedContainerInternal(
+ RMContainer rmContainer, ContainerStatus containerStatus,
+ RMContainerEventType event) {
+ completeContainerInternal(rmContainer, containerStatus, event);
+ }
+
+ private void completeContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
writeLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 8396db5..76648c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -836,7 +836,7 @@ public class FifoScheduler extends
@Lock(FifoScheduler.class)
@Override
- protected synchronized void completedContainerInternal(
+ protected synchronized void completeGuaranteedContainerInternal(
RMContainer rmContainer, ContainerStatus containerStatus,
RMContainerEventType event) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d217ee3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 7fbf84a..1ac20f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3194,6 +3194,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("unexpected container execution type",
ExecutionType.GUARANTEED,
allocatedContainers1.get(0).getExecutionType());
+ assertEquals(1,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
// node utilization is low after the container is launched on the node
ContainerStatus containerStatus = ContainerStatus.newInstance(
@@ -3235,6 +3237,29 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// OPPORTUNISTIC container allocation.
assertTrue("No reservation should be made.",
scheduler.getNode(node.getNodeID()).getReservedContainer() == null);
+
+ assertEquals(3,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
+ assertEquals(3200 + 512 + 1024,
+ scheduler.getRootQueueMetrics().getAllocatedMB());
+ assertEquals(3,
+ scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
+
+ // now the OPPORTUNISTIC container finishes
+ List<ContainerStatus> finishedContainers = Collections.singletonList(
+ ContainerStatus.newInstance(allocatedContainers3.get(0).getId(),
+ ContainerState.COMPLETE, "", ContainerExitStatus.SUCCESS));
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.emptyList(), finishedContainers),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+
+ assertEquals(2,
+ scheduler.getRootQueueMetrics().getAllocatedContainers());
+ assertEquals(3200 + 512,
+ scheduler.getRootQueueMetrics().getAllocatedMB());
+ assertEquals(2,
+ scheduler.getRootQueueMetrics().getAllocatedVirtualCores());
} finally {
conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
false);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org