You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by rk...@apache.org on 2017/10/24 17:25:35 UTC
hadoop git commit: YARN-7382. NoSuchElementException in FairScheduler
after failover causes RM crash (rkanter)
Repository: hadoop
Updated Branches:
refs/heads/trunk 679f99b14 -> 025c65657
YARN-7382. NoSuchElementException in FairScheduler after failover causes RM crash (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/025c6565
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/025c6565
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/025c6565
Branch: refs/heads/trunk
Commit: 025c6565725c1819566377632753e8b9055617a6
Parents: 679f99b
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Oct 24 10:21:44 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Oct 24 10:21:44 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/FSAppAttempt.java | 10 ++++++++++
.../TestWorkPreservingRMRestart.java | 21 +++++++++++++++++---
2 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025c6565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 8ab6e13..157d264 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -658,6 +658,16 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
getQueue().incUsedResource(rmContainer.getContainer().getResource());
}
+
+ // If not running unmanaged, the first container we recover is always
+ // the AM. Set the amResource for this app and update the leaf queue's AM
+ // usage
+ if (!isAmRunning() && !getUnmanagedAM()) {
+ Resource resource = rmContainer.getAllocatedResource();
+ setAMResource(resource);
+ getQueue().addAMResourceUsage(resource);
+ setAmRunning(true);
+ }
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/025c6565/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index c1cb4c1..064e217 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueu
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerTestBase;
@@ -158,6 +159,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
nm1.registerNode();
RMApp app1 = rm1.submitApp(200);
+ Resource amResources = app1.getAMResourceRequests().get(0).getCapability();
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
// clear queue metrics
@@ -240,7 +242,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
if (getSchedulerType() == SchedulerType.CAPACITY) {
checkCSQueue(rm2, schedulerApp, nmResource, nmResource, usedResources, 2);
} else {
- checkFSQueue(rm2, schedulerApp, usedResources, availableResources);
+ checkFSQueue(rm2, schedulerApp, usedResources, availableResources,
+ amResources);
}
// *********** check scheduler attempt state.********
@@ -310,6 +313,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
RMApp app1 = rm1.submitApp(200, "dynamicQApp",
UserGroupInformation.getCurrentUser().getShortUserName(), null,
ReservationSystemTestUtil.getReservationQueueName());
+ Resource amResources = app1.getAMResourceRequests().get(0).getCapability();
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
// clear queue metrics
@@ -384,7 +388,8 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
if (getSchedulerType() == SchedulerType.CAPACITY) {
checkCSQueue(rm2, schedulerApp, nmResource, nmResource, usedResources, 2);
} else {
- checkFSQueue(rm2, schedulerApp, usedResources, availableResources);
+ checkFSQueue(rm2, schedulerApp, usedResources, availableResources,
+ amResources);
}
// *********** check scheduler attempt state.********
@@ -456,7 +461,7 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
private void checkFSQueue(ResourceManager rm,
SchedulerApplication schedulerApp, Resource usedResources,
- Resource availableResources) throws Exception {
+ Resource availableResources, Resource amResources) throws Exception {
// waiting for RM's scheduling apps
int retry = 0;
Resource assumedFairShare = Resource.newInstance(8192, 8);
@@ -488,6 +493,16 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemorySize(),
availableResources.getVirtualCores(), usedResources.getMemorySize(),
usedResources.getVirtualCores());
+
+ // ************ check AM resources ****************
+ assertEquals(amResources,
+ schedulerApp.getCurrentAppAttempt().getAMResource());
+ FSQueueMetrics fsQueueMetrics =
+ (FSQueueMetrics) schedulerApp.getQueue().getMetrics();
+ assertEquals(amResources.getMemorySize(),
+ fsQueueMetrics.getAMResourceUsageMB());
+ assertEquals(amResources.getVirtualCores(),
+ fsQueueMetrics.getAMResourceUsageVCores());
}
// create 3 container reports for AM
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org