You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/01 22:53:15 UTC
[1/2] hadoop git commit: YARN-3222. Fixed NPE on
RMNodeImpl#ReconnectNodeTransition when a node is reconnected with a
different port. Contributed by Rohith Sharmaks (cherry picked from commit
b2f1ec312ee431aef762cfb49cb29cd6f4661e86)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6.1 6e090bc53 -> 2692401df
YARN-3222. Fixed NPE on RMNodeImpl#ReconnectNodeTransition when a node is reconnected with a different port. Contributed by Rohith Sharmaks
(cherry picked from commit b2f1ec312ee431aef762cfb49cb29cd6f4661e86)
(cherry picked from commit 888a44563819ba910dc3cc10d10ee0fb8f05db61)
(cherry picked from commit b78f87825bd593e30b2f2ea76f37c7a4fd673ab2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db92b09e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db92b09e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db92b09e
Branch: refs/heads/branch-2.6.1
Commit: db92b09e0396d4f0e5b227c40a1c0a8ca62eb048
Parents: 6e090bc
Author: Jian He <ji...@apache.org>
Authored: Tue Mar 3 16:25:57 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 1 13:39:35 2015 -0700
----------------------------------------------------------------------
.../resourcemanager/rmnode/RMNodeImpl.java | 34 +++++++++++---------
.../yarn/server/resourcemanager/MockNM.java | 6 +++-
.../TestResourceTrackerService.java | 17 +++++++++-
3 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db92b09e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 1774eb5..b92f399 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -564,12 +564,12 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
rmNode.nodeUpdateQueue.clear();
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeRemovedSchedulerEvent(rmNode));
-
+
if (rmNode.getHttpPort() == newNode.getHttpPort()) {
// Reset heartbeat ID since node just restarted.
rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
- if (rmNode.getState() != NodeState.UNHEALTHY) {
- // Only add new node if old state is not UNHEALTHY
+ if (rmNode.getState().equals(NodeState.RUNNING)) {
+ // Only add new node if old state is RUNNING
rmNode.context.getDispatcher().getEventHandler().handle(
new NodeAddedSchedulerEvent(newNode));
}
@@ -590,28 +590,30 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
} else {
rmNode.httpPort = newNode.getHttpPort();
rmNode.httpAddress = newNode.getHttpAddress();
- rmNode.totalCapability = newNode.getTotalCapability();
+ boolean isCapabilityChanged = false;
+ if (rmNode.getTotalCapability() != newNode.getTotalCapability()) {
+ rmNode.totalCapability = newNode.getTotalCapability();
+ isCapabilityChanged = true;
+ }
// Reset heartbeat ID since node just restarted.
rmNode.getLastNodeHeartBeatResponse().setResponseId(0);
- }
- if (null != reconnectEvent.getRunningApplications()) {
for (ApplicationId appId : reconnectEvent.getRunningApplications()) {
handleRunningAppOnNode(rmNode, rmNode.context, appId, rmNode.nodeId);
}
- }
- rmNode.context.getDispatcher().getEventHandler().handle(
- new NodesListManagerEvent(
- NodesListManagerEventType.NODE_USABLE, rmNode));
- if (rmNode.getState().equals(NodeState.RUNNING)) {
- // Update scheduler node's capacity for reconnect node.
- rmNode.context.getDispatcher().getEventHandler().handle(
- new NodeResourceUpdateSchedulerEvent(rmNode,
- ResourceOption.newInstance(newNode.getTotalCapability(), -1)));
+ if (isCapabilityChanged
+ && rmNode.getState().equals(NodeState.RUNNING)) {
+ // Update scheduler node's capacity for reconnect node.
+ rmNode.context
+ .getDispatcher()
+ .getEventHandler()
+ .handle(
+ new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption
+ .newInstance(newNode.getTotalCapability(), -1)));
+ }
}
-
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db92b09e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index 5f53805..c917f79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -51,7 +51,7 @@ public class MockNM {
private final int memory;
private final int vCores;
private ResourceTrackerService resourceTracker;
- private final int httpPort = 2;
+ private int httpPort = 2;
private MasterKey currentContainerTokenMasterKey;
private MasterKey currentNMTokenMasterKey;
private String version;
@@ -87,6 +87,10 @@ public class MockNM {
return httpPort;
}
+ public void setHttpPort(int port) {
+ httpPort = port;
+ }
+
public void setResourceTrackerService(ResourceTrackerService resourceTracker) {
this.resourceTracker = resourceTracker;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db92b09e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 009b9f4..7f173ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -621,7 +622,7 @@ public class TestResourceTrackerService {
dispatcher.await();
Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
-
+
// reconnect of node with changed capability and running applications
List<ApplicationId> runningApps = new ArrayList<ApplicationId>();
runningApps.add(ApplicationId.newInstance(1, 0));
@@ -631,6 +632,20 @@ public class TestResourceTrackerService {
dispatcher.await();
Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
Assert.assertEquals(5120 + 15360, metrics.getAvailableMB());
+
+ // reconnect healthy node changing http port
+ nm1 = new MockNM("host1:1234", 5120, rm.getResourceTrackerService());
+ nm1.setHttpPort(3);
+ nm1.registerNode();
+ dispatcher.await();
+ response = nm1.nodeHeartbeat(true);
+ response = nm1.nodeHeartbeat(true);
+ dispatcher.await();
+ RMNode rmNode = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+ Assert.assertEquals(3, rmNode.getHttpPort());
+ Assert.assertEquals(5120, rmNode.getTotalCapability().getMemory());
+ Assert.assertEquals(5120 + 15360, metrics.getAvailableMB());
+
}
private void writeToHostsFile(String... hosts) throws IOException {
[2/2] hadoop git commit: YARN-3222. Added the missing CHANGES.txt
entry.
Posted by vi...@apache.org.
YARN-3222. Added the missing CHANGES.txt entry.
(cherry picked from commit 4620767156ecc43424bc6c7c4d50519e2563cc69)
(cherry picked from commit 015696fb81912d545ce602b245c456c2741a3922)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2692401d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2692401d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2692401d
Branch: refs/heads/branch-2.6.1
Commit: 2692401dfbb49782cae8338e5f6c623a4541d96b
Parents: db92b09
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Sep 1 13:43:10 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 1 13:49:04 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2692401d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 78d7093..78c191f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -88,6 +88,9 @@ Release 2.6.1 - UNRELEASED
YARN-3239. WebAppProxy does not support a final tracking url which has
query fragments and params (Jian He via jlowe)
+ YARN-3222. Fixed RMNode to send scheduler events in sequential order when a
+ node reconnects. (Rohith Sharma K S via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES