You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/05/27 21:51:28 UTC
hadoop git commit: YARN-3647. RMWebServices api's should use updated
api from CommonNodeLabelsManager to get NodeLabel object. (Sunil G via
wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk c46d4bafe -> ec0a852a3
YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager to get NodeLabel object. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec0a852a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec0a852a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec0a852a
Branch: refs/heads/trunk
Commit: ec0a852a37d5c91a62d3d0ff3ddbd9d58235b312
Parents: c46d4ba
Author: Wangda Tan <wa...@apache.org>
Authored: Wed May 27 12:51:20 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed May 27 12:51:20 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../nodelabels/CommonNodeLabelsManager.java | 17 ++++---
.../nodelabels/RMNodeLabelsManager.java | 1 +
.../resourcemanager/webapp/RMWebServices.java | 27 +++++------
.../webapp/TestRMWebServicesNodeLabels.java | 51 +++++++++++++-------
5 files changed, 62 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7d122ff..4c75a39 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -265,6 +265,9 @@ Release 2.8.0 - UNRELEASED
YARN-3594. WintuilsProcessStubExecutor.startStreamReader leaks streams.
(Lars Francke via junping_du)
+ YARN-3647. RMWebServices api's should use updated api from CommonNodeLabelsManager
+ to get NodeLabel object. (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 2d57ad6..f741f68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -993,13 +993,18 @@ public class CommonNodeLabelsManager extends AbstractService {
}
}
- private Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
- Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
- if (labels.isEmpty()) {
- return EMPTY_NODELABEL_SET;
+ public Set<NodeLabel> getLabelsInfoByNode(NodeId nodeId) {
+ try {
+ readLock.lock();
+ Set<String> labels = getLabelsByNode(nodeId, nodeCollections);
+ if (labels.isEmpty()) {
+ return EMPTY_NODELABEL_SET;
+ }
+ Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
+ return nodeLabels;
+ } finally {
+ readLock.unlock();
}
- Set<NodeLabel> nodeLabels = createNodeLabelFromLabelNames(labels);
- return nodeLabels;
}
private Set<NodeLabel> createNodeLabelFromLabelNames(Set<String> labels) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 696b99b..10092ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -34,6 +34,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.RMNodeLabel;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index c39f2b9..b1fa80a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -810,12 +810,12 @@ public class RMWebServices {
NodeToLabelsInfo ntl = new NodeToLabelsInfo();
HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
- Map<NodeId, Set<String>> nodeIdToLabels = rm.getRMContext()
- .getNodeLabelManager().getNodeLabels();
+ Map<NodeId, Set<NodeLabel>> nodeIdToLabels = rm.getRMContext()
+ .getNodeLabelManager().getNodeLabelsInfo();
- for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
- ntlMap.put(nitle.getKey().toString(),
- new NodeLabelsInfo(nitle.getValue()));
+ for (Map.Entry<NodeId, Set<NodeLabel>> nitle : nodeIdToLabels.entrySet()) {
+ List<NodeLabel> labels = new ArrayList<NodeLabel>(nitle.getValue());
+ ntlMap.put(nitle.getKey().toString(), new NodeLabelsInfo(labels));
}
return ntl;
@@ -830,16 +830,16 @@ public class RMWebServices {
LabelsToNodesInfo lts = new LabelsToNodesInfo();
Map<NodeLabelInfo, NodeIDsInfo> ltsMap = lts.getLabelsToNodes();
- Map<String, Set<NodeId>> labelsToNodeId = null;
+ Map<NodeLabel, Set<NodeId>> labelsToNodeId = null;
if (labels == null || labels.size() == 0) {
labelsToNodeId =
- rm.getRMContext().getNodeLabelManager().getLabelsToNodes();
+ rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes();
} else {
labelsToNodeId =
- rm.getRMContext().getNodeLabelManager().getLabelsToNodes(labels);
+ rm.getRMContext().getNodeLabelManager().getLabelsInfoToNodes(labels);
}
- for (Entry<String, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
+ for (Entry<NodeLabel, Set<NodeId>> entry : labelsToNodeId.entrySet()) {
List<String> nodeIdStrList = new ArrayList<String>();
for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString());
@@ -985,14 +985,13 @@ public class RMWebServices {
@Path("/nodes/{nodeId}/get-labels")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
- @PathParam("nodeId") String nodeId)
- throws IOException {
+ @PathParam("nodeId") String nodeId) throws IOException {
init();
NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
- return new NodeLabelsInfo(
- rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid));
-
+ List<NodeLabel> labels = new ArrayList<NodeLabel>(rm.getRMContext()
+ .getNodeLabelManager().getLabelsInfoByNode(nid));
+ return new NodeLabelsInfo(labels);
}
protected Response killApp(RMApp app, UserGroupInformation callerUGI,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0a852a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index 54fe00d..53a9902 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -133,12 +133,15 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size());
+ for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+ assertEquals("a", nl.getName());
+ assertTrue(nl.getExclusivity());
+ }
// Add another
nlsifo = new NodeLabelsInfo();
- nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b"));
+ nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("b", false));
response =
r.path("ws").path("v1").path("cluster")
.path("add-node-labels").queryParam("user.name", userName)
@@ -154,6 +157,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals(2, nlsifo.getNodeLabels().size());
+ // Verify exclusivity for 'y' as false
+ for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+ if (nl.getName().equals("b")) {
+ assertFalse(nl.getExclusivity());
+ }
+ }
// Add labels to a node
MultivaluedMapImpl params = new MultivaluedMapImpl();
@@ -202,7 +211,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
assertEquals(2, ltni.getLabelsToNodes().size());
- NodeIDsInfo nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("b"));
+ NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
+ new NodeLabelInfo("b", false));
assertTrue(nodes.getNodeIDs().contains("nid2:0"));
assertTrue(nodes.getNodeIDs().contains("nid1:0"));
nodes = ltni.getLabelsToNodes().get(new NodeLabelInfo("a"));
@@ -231,7 +241,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertTrue(nlsifo.getNodeLabelsName().contains("a"));
+ assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Replace
@@ -255,7 +265,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertTrue(nlsifo.getNodeLabelsName().contains("b"));
+ assertTrue(nlsifo.getNodeLabelsInfo().contains(
+ new NodeLabelInfo("b", false)));
// Replace labels using node-to-labels
NodeToLabelsEntryList ntli = new NodeToLabelsEntryList();
@@ -281,7 +292,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
NodeToLabelsInfo ntlinfo = response.getEntity(NodeToLabelsInfo.class);
NodeLabelsInfo nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
- assertTrue(nlinfo.getNodeLabelsName().contains("a"));
+ assertTrue(nlinfo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Remove all
params = new MultivaluedMapImpl();
@@ -303,7 +314,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertTrue(nlsifo.getNodeLabelsName().contains(""));
+ assertTrue(nlsifo.getNodeLabelsInfo().isEmpty());
// Add a label back for auth tests
params = new MultivaluedMapImpl();
@@ -326,7 +337,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertTrue(nlsifo.getNodeLabelsName().contains("a"));
+ assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Auth fail replace labels on node
params = new MultivaluedMapImpl();
@@ -347,7 +358,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertTrue(nlsifo.getNodeLabelsName().contains("a"));
+ assertTrue(nlsifo.getNodeLabelsInfo().contains(new NodeLabelInfo("a")));
// Fail to add a label with post
response =
@@ -383,8 +394,11 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
- assertEquals("a", nlsifo.getNodeLabelsInfo().get(0).getName());
assertEquals(1, nlsifo.getNodeLabels().size());
+ for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+ assertEquals("a", nl.getName());
+ assertTrue(nl.getExclusivity());
+ }
// Remove cluster label with post
params = new MultivaluedMapImpl();
@@ -409,8 +423,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
// configuration is on
// Reset for testing : add cluster labels
nlsifo = new NodeLabelsInfo();
- nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x"));
- nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
+ nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("x", false));
+ nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y", false));
response =
r.path("ws")
.path("v1")
@@ -421,8 +435,6 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.entity(toJson(nlsifo, NodeLabelsInfo.class),
MediaType.APPLICATION_JSON).post(ClientResponse.class);
// Reset for testing : Add labels to a node
- nlsifo = new NodeLabelsInfo();
- nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("y"));
params = new MultivaluedMapImpl();
params.add("labels", "y");
response =
@@ -461,7 +473,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
- assertFalse(nlinfo.getNodeLabels().contains("x"));
+ assertFalse(nlinfo.getNodeLabelsInfo().contains(
+ new NodeLabelInfo("x", false)));
// Case2 : failure to Replace labels using replace-labels
response =
@@ -481,7 +494,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
ntlinfo = response.getEntity(NodeToLabelsInfo.class);
nlinfo = ntlinfo.getNodeToLabels().get("nid:0");
assertEquals(1, nlinfo.getNodeLabels().size());
- assertFalse(nlinfo.getNodeLabels().contains("x"));
+ assertFalse(nlinfo.getNodeLabelsInfo().contains(
+ new NodeLabelInfo("x", false)));
// Case3 : Remove cluster label should be successful
params = new MultivaluedMapImpl();
@@ -500,7 +514,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
+ assertEquals(new NodeLabelInfo("y", false),
+ nlsifo.getNodeLabelsInfo().get(0));
assertEquals("y", nlsifo.getNodeLabelsInfo().get(0).getName());
+ assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
// Remove y
params = new MultivaluedMapImpl();
@@ -543,7 +560,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
nlsifo = response.getEntity(NodeLabelsInfo.class);
assertEquals("z", nlsifo.getNodeLabelsInfo().get(0).getName());
- assertEquals(false, nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
+ assertFalse(nlsifo.getNodeLabelsInfo().get(0).getExclusivity());
assertEquals(1, nlsifo.getNodeLabels().size());
}