You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sa...@apache.org on 2014/05/14 08:44:52 UTC
svn commit: r1594478 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/
hadoop-yarn/hadoop-yarn-ser...
Author: sandy
Date: Wed May 14 06:44:52 2014
New Revision: 1594478
URL: http://svn.apache.org/r1594478
Log:
YARN-1986. In Fifo Scheduler, node heartbeat in between creating app and attempt causes NPE (Hong Zhiguo via Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1594478&r1=1594477&r2=1594478&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed May 14 06:44:52 2014
@@ -207,6 +207,9 @@ Release 2.4.1 - UNRELEASED
YARN-1957. Consider the max capacity of the queue when computing the ideal
capacity for preemption. (Carlo Curino via cdouglas)
+ YARN-1986. In Fifo Scheduler, node heartbeat in between creating app and
+ attempt causes NPE (Hong Zhiguo via Sandy Ryza)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1594478&r1=1594477&r2=1594478&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Wed May 14 06:44:52 2014
@@ -360,7 +360,8 @@ public class FifoScheduler extends Abstr
return nodes.get(nodeId);
}
- private synchronized void addApplication(ApplicationId applicationId,
+ @VisibleForTesting
+ public synchronized void addApplication(ApplicationId applicationId,
String queue, String user) {
SchedulerApplication application =
new SchedulerApplication(DEFAULT_QUEUE, user);
@@ -372,7 +373,8 @@ public class FifoScheduler extends Abstr
.handle(new RMAppEvent(applicationId, RMAppEventType.APP_ACCEPTED));
}
- private synchronized void
+ @VisibleForTesting
+ public synchronized void
addApplicationAttempt(ApplicationAttemptId appAttemptId,
boolean transferStateFromPreviousAttempt) {
SchedulerApplication application =
@@ -458,6 +460,9 @@ public class FifoScheduler extends Abstr
.entrySet()) {
FiCaSchedulerApp application =
(FiCaSchedulerApp) e.getValue().getCurrentAppAttempt();
+ if (application == null) {
+ continue;
+ }
LOG.debug("pre-assignContainers");
application.showRequests();
synchronized (application) {
@@ -497,6 +502,9 @@ public class FifoScheduler extends Abstr
for (SchedulerApplication application : applications.values()) {
FiCaSchedulerApp attempt =
(FiCaSchedulerApp) application.getCurrentAppAttempt();
+ if (attempt == null) {
+ continue;
+ }
attempt.setHeadroom(Resources.subtract(clusterResource, usedResource));
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1594478&r1=1594477&r2=1594478&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Wed May 14 06:44:52 2014
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -66,7 +67,7 @@ public class TestFifoScheduler {
private final int GB = 1024;
private static YarnConfiguration conf;
-
+
@BeforeClass
public static void setup() {
conf = new YarnConfiguration();
@@ -213,6 +214,32 @@ public class TestFifoScheduler {
rm.stop();
}
+ @Test
+ public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
+ FifoScheduler scheduler = new FifoScheduler();
+ MockRM rm = new MockRM(conf);
+ scheduler.reinitialize(conf, rm.getRMContext());
+
+ RMNode node = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024, 4), 1, "127.0.0.1");
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ scheduler.addApplication(appId, "queue1", "user1");
+
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ try {
+ scheduler.handle(updateEvent);
+ } catch (NullPointerException e) {
+ Assert.fail();
+ }
+
+ ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
+ scheduler.addApplicationAttempt(attId, false);
+
+ rm.stop();
+ }
+
private void testMinimumAllocation(YarnConfiguration conf, int testAlloc)
throws Exception {
MockRM rm = new MockRM(conf);