You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/11/09 17:54:28 UTC
[26/30] hadoop git commit: YARN-2505. Supported get/add/remove/change
labels in RM REST API. Contributed by Craig Welch.
YARN-2505. Supported get/add/remove/change labels in RM REST API. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a4e0d34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a4e0d34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a4e0d34
Branch: refs/heads/HDFS-EC
Commit: 9a4e0d343e9e891c10ef6682e7b2231a59e69ade
Parents: df36edf
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri Nov 7 20:35:46 2014 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri Nov 7 20:35:46 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/util/ConverterUtils.java | 7 +
.../hadoop/yarn/util/TestConverterUtils.java | 14 +
.../resourcemanager/webapp/RMWebServices.java | 180 +++++++++-
.../dao/ApplicationSubmissionContextInfo.java | 23 ++
.../webapp/dao/NodeLabelsInfo.java | 52 +++
.../webapp/dao/NodeToLabelsInfo.java | 41 +++
.../webapp/TestRMWebServicesNodeLabels.java | 357 +++++++++++++++++++
8 files changed, 676 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1e6406a..748ffe0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -198,6 +198,9 @@ Release 2.6.0 - UNRELEASED
YARN-2632. Document NM Restart feature. (Junping Du and Vinod Kumar
Vavilapalli via jlowe)
+ YARN-2505. Supported get/add/remove/change labels in RM REST API. (Craig Welch
+ via zjshen)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
index 27f7bc1..012d799 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
@@ -151,6 +151,13 @@ public class ConverterUtils {
public static String toString(ContainerId cId) {
return cId == null ? null : cId.toString();
}
+
+ public static NodeId toNodeIdWithDefaultPort(String nodeIdStr) {
+ if (nodeIdStr.indexOf(":") < 0) {
+ return toNodeId(nodeIdStr + ":0");
+ }
+ return toNodeId(nodeIdStr);
+ }
public static NodeId toNodeId(String nodeIdStr) {
String[] parts = nodeIdStr.split(":");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
index 824e6c0..7d53785 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestConverterUtils.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.TestContainerId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.junit.Test;
public class TestConverterUtils {
@@ -85,4 +86,17 @@ public class TestConverterUtils {
public void testContainerIdNull() throws URISyntaxException {
assertNull(ConverterUtils.toString((ContainerId)null));
}
+
+ @Test
+ public void testNodeIdWithDefaultPort() throws URISyntaxException {
+ NodeId nid;
+
+ nid = ConverterUtils.toNodeIdWithDefaultPort("node:10");
+ assertEquals(nid.getPort(), 10);
+ assertEquals(nid.getHost(), "node");
+
+ nid = ConverterUtils.toNodeIdWithDefaultPort("node");
+ assertEquals(nid.getPort(), 0);
+ assertEquals(nid.getHost(), "node");
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 87c895a..cf0a83a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -24,6 +24,7 @@ import java.security.AccessControlException;
import java.nio.ByteBuffer;
import java.security.Principal;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
@@ -133,6 +134,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -715,6 +718,179 @@ public class RMWebServices {
return Response.status(Status.OK).entity(ret).build();
}
+
+ @GET
+ @Path("/get-node-to-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public NodeToLabelsInfo getNodeToLabels(@Context HttpServletRequest hsr)
+ throws IOException {
+ init();
+
+ NodeToLabelsInfo ntl = new NodeToLabelsInfo();
+ HashMap<String, NodeLabelsInfo> ntlMap = ntl.getNodeToLabels();
+ Map<NodeId, Set<String>> nodeIdToLabels =
+ rm.getRMContext().getNodeLabelManager().getNodeLabels();
+
+ for (Map.Entry<NodeId, Set<String>> nitle : nodeIdToLabels.entrySet()) {
+ ntlMap.put(nitle.getKey().toString(),
+ new NodeLabelsInfo(nitle.getValue()));
+ }
+
+ return ntl;
+ }
+
+ @POST
+ @Path("/replace-node-to-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response replaceLabelsOnNodes(
+ final NodeToLabelsInfo newNodeToLabels,
+ @Context HttpServletRequest hsr)
+ throws IOException {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated for"
+ + " post to .../replace-node-to-labels";
+ throw new AuthorizationException(msg);
+ }
+ if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+ String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+ + " for post to .../replace-node-to-labels ";
+ throw new AuthorizationException(msg);
+ }
+
+ Map<NodeId, Set<String>> nodeIdToLabels =
+ new HashMap<NodeId, Set<String>>();
+
+ for (Map.Entry<String, NodeLabelsInfo> nitle :
+ newNodeToLabels.getNodeToLabels().entrySet()) {
+ nodeIdToLabels.put(ConverterUtils.toNodeIdWithDefaultPort(nitle.getKey()),
+ new HashSet<String>(nitle.getValue().getNodeLabels()));
+ }
+
+ rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeIdToLabels);
+
+ return Response.status(Status.OK).build();
+ }
+
+ @GET
+ @Path("/get-node-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public NodeLabelsInfo getClusterNodeLabels(@Context HttpServletRequest hsr)
+ throws IOException {
+ init();
+
+ NodeLabelsInfo ret =
+ new NodeLabelsInfo(rm.getRMContext().getNodeLabelManager()
+ .getClusterNodeLabels());
+
+ return ret;
+ }
+
+ @POST
+ @Path("/add-node-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response addToClusterNodeLabels(final NodeLabelsInfo newNodeLabels,
+ @Context HttpServletRequest hsr)
+ throws Exception {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated for"
+ + " post to .../add-node-labels";
+ throw new AuthorizationException(msg);
+ }
+ if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+ String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+ + " for post to .../add-node-labels ";
+ throw new AuthorizationException(msg);
+ }
+
+ rm.getRMContext().getNodeLabelManager()
+ .addToCluserNodeLabels(new HashSet<String>(
+ newNodeLabels.getNodeLabels()));
+
+ return Response.status(Status.OK).build();
+
+ }
+
+ @POST
+ @Path("/remove-node-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response removeFromCluserNodeLabels(final NodeLabelsInfo oldNodeLabels,
+ @Context HttpServletRequest hsr)
+ throws Exception {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated for"
+ + " post to .../remove-node-labels";
+ throw new AuthorizationException(msg);
+ }
+ if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+ String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+ + " for post to .../remove-node-labels ";
+ throw new AuthorizationException(msg);
+ }
+
+ rm.getRMContext().getNodeLabelManager()
+ .removeFromClusterNodeLabels(new HashSet<String>(
+ oldNodeLabels.getNodeLabels()));
+
+ return Response.status(Status.OK).build();
+
+ }
+
+ @GET
+ @Path("/nodes/{nodeId}/get-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public NodeLabelsInfo getLabelsOnNode(@Context HttpServletRequest hsr,
+ @PathParam("nodeId") String nodeId)
+ throws IOException {
+ init();
+
+ NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
+ return new NodeLabelsInfo(
+ rm.getRMContext().getNodeLabelManager().getLabelsOnNode(nid));
+
+ }
+
+ @POST
+ @Path("/nodes/{nodeId}/replace-labels")
+ @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+ public Response replaceLabelsOnNode(NodeLabelsInfo newNodeLabelsInfo,
+ @Context HttpServletRequest hsr, @PathParam("nodeId") String nodeId)
+ throws Exception {
+ init();
+
+ UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+ if (callerUGI == null) {
+ String msg = "Unable to obtain user name, user not authenticated for"
+ + " post to .../nodes/nodeid/replace-labels";
+ throw new AuthorizationException(msg);
+ }
+
+ if (!rm.getRMContext().getNodeLabelManager().checkAccess(callerUGI)) {
+ String msg = "User " + callerUGI.getShortUserName() + " not authorized"
+ + " for post to .../nodes/nodeid/replace-labels";
+ throw new AuthorizationException(msg);
+ }
+
+ NodeId nid = ConverterUtils.toNodeIdWithDefaultPort(nodeId);
+
+ Map<NodeId, Set<String>> newLabelsForNode = new HashMap<NodeId,
+ Set<String>>();
+
+ newLabelsForNode.put(nid, new HashSet<String>(newNodeLabelsInfo.getNodeLabels()));
+
+ rm.getRMContext().getNodeLabelManager().replaceLabelsOnNode(newLabelsForNode);
+
+ return Response.status(Status.OK).build();
+
+ }
protected Response killApp(RMApp app, UserGroupInformation callerUGI,
HttpServletRequest hsr) throws IOException, InterruptedException {
@@ -965,7 +1141,9 @@ public class RMWebServices {
newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
createAppSubmissionContextResource(newApp),
newApp.getApplicationType(),
- newApp.getKeepContainersAcrossApplicationAttempts());
+ newApp.getKeepContainersAcrossApplicationAttempts(),
+ newApp.getAppNodeLabelExpression(),
+ newApp.getAMContainerNodeLabelExpression());
appContext.setApplicationTags(newApp.getApplicationTags());
return appContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
index f7233e6..5278b3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ApplicationSubmissionContextInfo.java
@@ -71,6 +71,12 @@ public class ApplicationSubmissionContextInfo {
@XmlElementWrapper(name = "application-tags")
@XmlElement(name = "tag")
Set<String> tags;
+
+ @XmlElement(name = "app-node-label-expression")
+ String appNodeLabelExpression;
+
+ @XmlElement(name = "am-container-node-label-expression")
+ String amContainerNodeLabelExpression;
public ApplicationSubmissionContextInfo() {
applicationId = "";
@@ -83,6 +89,8 @@ public class ApplicationSubmissionContextInfo {
keepContainers = false;
applicationType = "";
tags = new HashSet<String>();
+ appNodeLabelExpression = "";
+ amContainerNodeLabelExpression = "";
}
public String getApplicationId() {
@@ -132,6 +140,14 @@ public class ApplicationSubmissionContextInfo {
public Set<String> getApplicationTags() {
return tags;
}
+
+ public String getAppNodeLabelExpression() {
+ return appNodeLabelExpression;
+ }
+
+ public String getAMContainerNodeLabelExpression() {
+ return amContainerNodeLabelExpression;
+ }
public void setApplicationId(String applicationId) {
this.applicationId = applicationId;
@@ -182,5 +198,12 @@ public class ApplicationSubmissionContextInfo {
public void setApplicationTags(Set<String> tags) {
this.tags = tags;
}
+
+ public void setAppNodeLabelExpression(String appNodeLabelExpression) {
+ this.appNodeLabelExpression = appNodeLabelExpression;
+ }
+ public void setAMContainerNodeLabelExpression(String nodeLabelExpression) {
+ this.amContainerNodeLabelExpression = nodeLabelExpression;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java
new file mode 100644
index 0000000..1cb895a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeLabelsInfo.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.*;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "nodeLabelsInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeLabelsInfo {
+
+ protected ArrayList<String> nodeLabels = new ArrayList<String>();
+
+ public NodeLabelsInfo() {
+ } // JAXB needs this
+
+ public NodeLabelsInfo(ArrayList<String> nodeLabels) {
+ this.nodeLabels = nodeLabels;
+ }
+
+ public NodeLabelsInfo(Set<String> nodeLabelsSet) {
+ this.nodeLabels = new ArrayList<String>(nodeLabelsSet);
+ }
+
+ public ArrayList<String> getNodeLabels() {
+ return nodeLabels;
+ }
+
+ public void setNodeLabels(ArrayList<String> nodeLabels) {
+ this.nodeLabels = nodeLabels;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java
new file mode 100644
index 0000000..f2e6441
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeToLabelsInfo.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.*;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "nodeToLabelsInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeToLabelsInfo {
+
+ protected HashMap<String, NodeLabelsInfo> nodeToLabels =
+ new HashMap<String, NodeLabelsInfo>();
+
+ public NodeToLabelsInfo() {
+ } // JAXB needs this
+
+ public HashMap<String, NodeLabelsInfo> getNodeToLabels() {
+ return nodeToLabels;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a4e0d34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
new file mode 100644
index 0000000..3c958f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -0,0 +1,357 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.StringWriter;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Test;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.json.JSONJAXBContext;
+import com.sun.jersey.api.json.JSONMarshaller;
+import com.sun.jersey.api.json.JSONUnmarshaller;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+public class TestRMWebServicesNodeLabels extends JerseyTest {
+
+ private static final Log LOG = LogFactory
+ .getLog(TestRMWebServicesNodeLabels.class);
+
+ private static MockRM rm;
+ private YarnConfiguration conf;
+
+ private String userName;
+ private String notUserName;
+
+ private Injector injector = Guice.createInjector(new ServletModule() {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ try {
+ userName = UserGroupInformation.getCurrentUser().getShortUserName();
+ } catch (IOException ioe) {
+ throw new RuntimeException("Unable to get current user name "
+ + ioe.getMessage(), ioe);
+ }
+ notUserName = userName + "abc123";
+ conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.YARN_ADMIN_ACL, userName);
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ bind(RMContext.class).toInstance(rm.getRMContext());
+ filter("/*").through(
+ TestRMWebServicesAppsModification.TestRMCustomAuthFilter.class);
+ serve("/*").with(GuiceContainer.class);
+ }
+ });
+
+ public class GuiceServletConfig extends GuiceServletContextListener {
+
+ @Override
+ protected Injector getInjector() {
+ return injector;
+ }
+ }
+
+ public TestRMWebServicesNodeLabels() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testNodeLabels() throws JSONException, Exception {
+ WebResource r = resource();
+
+ ClientResponse response;
+ JSONObject json;
+ JSONArray jarr;
+ String responseString;
+
+ // Add a label
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("add-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("a", json.getString("nodeLabels"));
+
+ // Add another
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("add-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ jarr = json.getJSONArray("nodeLabels");
+ assertEquals(2, jarr.length());
+
+ // Add labels to a node
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("replace-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+ MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ LOG.info("posted node nodelabel");
+
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("get-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ jarr = json.getJSONArray("nodeLabels");
+ assertEquals(2, jarr.length());
+
+ // Replace
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("replace-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ LOG.info("posted node nodelabel");
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("get-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("a", json.getString("nodeLabels"));
+
+ // Replace labels using node-to-labels
+ NodeToLabelsInfo ntli = new NodeToLabelsInfo();
+ NodeLabelsInfo nli = new NodeLabelsInfo();
+ nli.getNodeLabels().add("a");
+ nli.getNodeLabels().add("b");
+ ntli.getNodeToLabels().put("nid:0", nli);
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("replace-node-to-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(toJson(ntli, NodeToLabelsInfo.class),
+ MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+
+ // Verify, using node-to-labels
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-to-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ ntli = response.getEntity(NodeToLabelsInfo.class);
+ nli = ntli.getNodeToLabels().get("nid:0");
+ assertEquals(2, nli.getNodeLabels().size());
+ assertTrue(nli.getNodeLabels().contains("a"));
+ assertTrue(nli.getNodeLabels().contains("b"));
+
+ // Remove all
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("replace-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ LOG.info("posted node nodelabel");
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("get-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("", json.getString("nodeLabels"));
+
+ // Add a label back for auth tests
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("replace-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\": \"a\"}",
+ MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ LOG.info("posted node nodelabel");
+
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("get-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("a", json.getString("nodeLabels"));
+
+ // Auth fail replace labels on node
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("replace-labels")
+ .queryParam("user.name", notUserName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+ MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("nodes").path("nid:0")
+ .path("get-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("a", json.getString("nodeLabels"));
+
+ // Fail to add a label with post
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("add-node-labels").queryParam("user.name", notUserName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"c\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ jarr = json.getJSONArray("nodeLabels");
+ assertEquals(2, jarr.length());
+
+ // Remove cluster label (succeed, we no longer need it)
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("remove-node-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("a", json.getString("nodeLabels"));
+
+
+ // Remove cluster label with post
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("remove-node-labels")
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ // Verify
+ response =
+ r.path("ws").path("v1").path("cluster")
+ .path("get-node-labels").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ String res = response.getEntity(String.class);
+ assertTrue(res.equals("null"));
+ }
+
+ @SuppressWarnings("rawtypes")
+ private String toJson(Object nsli, Class klass) throws Exception {
+ StringWriter sw = new StringWriter();
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONMarshaller jm = ctx.createJSONMarshaller();
+ jm.marshallToJSON(nsli, sw);
+ return sw.toString();
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ private Object fromJson(String json, Class klass) throws Exception {
+ StringReader sr = new StringReader(json);
+ JSONJAXBContext ctx = new JSONJAXBContext(klass);
+ JSONUnmarshaller jm = ctx.createJSONUnmarshaller();
+ return jm.unmarshalFromJSON(sr, klass);
+ }
+
+}