You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2020/10/22 07:23:19 UTC
[hadoop] branch trunk updated: YARN-10453. Add partition resource
info to get-node-labels and label-mappings api responses. Contributed Akhil
PB.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7435604 YARN-10453. Add partition resource info to get-node-labels and label-mappings api responses. Contributed Akhil PB.
7435604 is described below
commit 7435604a91a49f5c5717083fbaee74dd8ec1c426
Author: Sunil G <su...@apache.org>
AuthorDate: Tue Oct 20 15:53:44 2020 +0530
YARN-10453. Add partition resource info to get-node-labels and label-mappings api responses. Contributed Akhil PB.
---
.../server/resourcemanager/webapp/NodeIDsInfo.java | 16 ++++++
.../resourcemanager/webapp/RMWebServices.java | 18 +++++-
.../resourcemanager/webapp/dao/NodeLabelInfo.java | 10 ++++
.../{NodeIDsInfo.java => dao/PartitionInfo.java} | 30 ++++------
.../webapp/TestRMWebServicesNodeLabels.java | 67 +++++++++++++++++++++-
5 files changed, 118 insertions(+), 23 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
index 5f45b96..b492793 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
+
import java.util.ArrayList;
import java.util.List;
@@ -39,6 +43,9 @@ public class NodeIDsInfo {
@XmlElement(name="nodes")
protected ArrayList<String> nodeIDsList = new ArrayList<String>();
+ @XmlElement(name = "partitionInfo")
+ private PartitionInfo partitionInfo;
+
public NodeIDsInfo() {
} // JAXB needs this
@@ -46,7 +53,16 @@ public class NodeIDsInfo {
this.nodeIDsList.addAll(nodeIdsList);
}
+ public NodeIDsInfo(List<String> nodeIdsList, Resource resource) {
+ this(nodeIdsList);
+ this.partitionInfo = new PartitionInfo(new ResourceInfo(resource));
+ }
+
public ArrayList<String> getNodeIDs() {
return nodeIDsList;
}
+
+ public PartitionInfo getPartitionInfo() {
+ return partitionInfo;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 02ad3ab..2a725cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -184,6 +184,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntr
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.RMQueueAclInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo;
@@ -209,6 +210,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
import org.apache.hadoop.yarn.util.AdHocLogDumper;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -1296,8 +1298,10 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
for (NodeId nodeId : entry.getValue()) {
nodeIdStrList.add(nodeId.toString());
}
+ Resource resource = rm.getRMContext().getNodeLabelManager()
+ .getResourceByLabel(entry.getKey().getName(), Resources.none());
ltsMap.put(new NodeLabelInfo(entry.getKey()),
- new NodeIDsInfo(nodeIdStrList));
+ new NodeIDsInfo(nodeIdStrList, resource));
}
return lts;
}
@@ -1386,9 +1390,17 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
List<NodeLabel> nodeLabels =
rm.getRMContext().getNodeLabelManager().getClusterNodeLabels();
- NodeLabelsInfo ret = new NodeLabelsInfo(nodeLabels);
- return ret;
+ ArrayList<NodeLabelInfo> nodeLabelsInfo = new ArrayList<NodeLabelInfo>();
+ for (NodeLabel label: nodeLabels) {
+ Resource resource = rm.getRMContext().getNodeLabelManager()
+ .getResourceByLabel(label.getName(), Resources.none());
+ PartitionInfo partitionInfo =
+ new PartitionInfo(new ResourceInfo(resource));
+ nodeLabelsInfo.add(new NodeLabelInfo(label, partitionInfo));
+ }
+
+ return new NodeLabelsInfo(nodeLabelsInfo);
}
@POST
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java
index e507c46..ce47471 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelInfo.java
@@ -30,6 +30,7 @@ public class NodeLabelInfo {
private String name;
private boolean exclusivity;
+ private PartitionInfo partitionInfo;
public NodeLabelInfo() {
// JAXB needs this
@@ -50,6 +51,11 @@ public class NodeLabelInfo {
this.exclusivity = label.isExclusive();
}
+ public NodeLabelInfo(NodeLabel label, PartitionInfo partitionInfo) {
+ this(label);
+ this.partitionInfo = partitionInfo;
+ }
+
public String getName() {
return name;
}
@@ -58,6 +64,10 @@ public class NodeLabelInfo {
return exclusivity;
}
+ public PartitionInfo getPartitionInfo() {
+ return partitionInfo;
+ }
+
@Override
public boolean equals(Object obj) {
if (this == obj) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java
similarity index 63%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java
index 5f45b96..95baf74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeIDsInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionInfo.java
@@ -16,10 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-
-import java.util.ArrayList;
-import java.util.List;
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -27,26 +24,23 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
/**
- * XML element uses to represent NodeIds' list.
+ * XML element uses to represent partitionInfo.
*/
-@XmlRootElement(name = "nodeIDsInfo")
+@XmlRootElement(name = "partitionInfo")
@XmlAccessorType(XmlAccessType.FIELD)
-public class NodeIDsInfo {
+public class PartitionInfo {
- /**
- * Set doesn't support default no arg constructor which is req by JAXB
- */
- @XmlElement(name="nodes")
- protected ArrayList<String> nodeIDsList = new ArrayList<String>();
+ @XmlElement(name = "resourceAvailable")
+ private ResourceInfo resourceAvailable;
- public NodeIDsInfo() {
- } // JAXB needs this
+ public PartitionInfo() {
+ }
- public NodeIDsInfo(List<String> nodeIdsList) {
- this.nodeIDsList.addAll(nodeIdsList);
+ public PartitionInfo(ResourceInfo resourceAvailable) {
+ this.resourceAvailable = resourceAvailable;
}
- public ArrayList<String> getNodeIDs() {
- return nodeIDsList;
+ public ResourceInfo getResourceAvailable() {
+ return resourceAvailable;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index 0b2c6fe..f1ea0fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
@@ -130,7 +131,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("add-node-labels").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON)
+ .entity(toJson(nlsifo, NodeLabelsInfo.class),
+ MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
// Verify
@@ -154,7 +156,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
r.path("ws").path("v1").path("cluster")
.path("add-node-labels").queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON)
+ .entity(toJson(nlsifo, NodeLabelsInfo.class),
+ MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
// Verify
@@ -690,6 +693,66 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
validateJsonExceptionContent(response, expectedmessage);
}
+ @Test
+ public void testNodeLabelPartitionInfo() throws Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+
+ // Add a node label
+ NodeLabelsInfo nlsifo = new NodeLabelsInfo();
+ nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("a"));
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("add-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(nlsifo, NodeLabelsInfo.class), MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+
+ // Verify partition info in get-node-labels
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ nlsifo = response.getEntity(NodeLabelsInfo.class);
+ assertEquals(1, nlsifo.getNodeLabels().size());
+ for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) {
+ assertEquals("a", nl.getName());
+ assertTrue(nl.getExclusivity());
+ assertNotNull(nl.getPartitionInfo());
+ assertNotNull(nl.getPartitionInfo().getResourceAvailable());
+ }
+
+ // Add node label to a node
+ MultivaluedMapImpl params = new MultivaluedMapImpl();
+ params.add("labels", "a");
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nodeId:0")
+ .path("replace-labels")
+ .queryParam("user.name", userName)
+ .queryParams(params)
+ .accept(MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+
+ // Verify partition info in label-mappings
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("label-mappings").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ LabelsToNodesInfo ltni = response.getEntity(LabelsToNodesInfo.class);
+ assertEquals(1, ltni.getLabelsToNodes().size());
+ NodeIDsInfo nodes = ltni.getLabelsToNodes().get(
+ new NodeLabelInfo("a"));
+ assertTrue(nodes.getNodeIDs().contains("nodeId:0"));
+ assertNotNull(nodes.getPartitionInfo());
+ assertNotNull(nodes.getPartitionInfo().getResourceAvailable());
+ }
+
@SuppressWarnings("rawtypes")
private String toJson(Object nsli, Class klass) throws Exception {
StringWriter sw = new StringWriter();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org