You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/06/08 02:55:25 UTC
svn commit: r1490892 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/
hadoop-yarn/hadoop-yarn-server/hado...
Author: vinodkv
Date: Sat Jun 8 00:55:25 2013
New Revision: 1490892
URL: http://svn.apache.org/r1490892
Log:
YARN-642. Removed health parameter from ResourceManager /nodes web-service and cleaned the behaviour of the status parameter. Contributed by Sandy Ryza.
svn merge --ignore-ancestry -c 1490890 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1490892&r1=1490891&r2=1490892&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Sat Jun 8 00:55:25 2013
@@ -109,6 +109,9 @@ Release 2.1.0-beta - UNRELEASED
YARN-777. Removed unreferenced objects from .proto files. (Jian He via
vinodkv)
+ YARN-642. Removed health parameter from ResourceManager /nodes web-service
+ and cleaned the behaviour of the status parameter. (Sandy Ryza vid vinodkv)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1490892&r1=1490891&r2=1490892&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Sat Jun 8 00:55:25 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
-import java.util.Collection;
import java.util.concurrent.ConcurrentMap;
import javax.servlet.http.HttpServletRequest;
@@ -32,8 +31,6 @@ import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -75,7 +72,6 @@ import com.google.inject.Singleton;
@Path("/ws/v1/cluster")
public class RMWebServices {
private static final String EMPTY = "";
- private static final Log LOG = LogFactory.getLog(RMWebServices.class);
private final ResourceManager rm;
private static RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
@@ -152,60 +148,57 @@ public class RMWebServices {
return new SchedulerTypeInfo(sinfo);
}
+ /**
+ * If no params are given, returns all active nodes, which includes
+ * nodes in the NEW and RUNNING states. If state param is "all", returns all
+ * nodes in all states. Otherwise, if the state param is set to a state name,
+ * returns all nodes that are in that state.
+ */
@GET
@Path("/nodes")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
- public NodesInfo getNodes(@QueryParam("state") String filterState,
- @QueryParam("healthy") String healthState) {
+ public NodesInfo getNodes(@QueryParam("state") String state) {
init();
ResourceScheduler sched = this.rm.getResourceScheduler();
if (sched == null) {
throw new NotFoundException("Null ResourceScheduler instance");
}
- Collection<RMNode> rmNodes = this.rm.getRMContext().getRMNodes().values();
- boolean isInactive = false;
- if (filterState != null && !filterState.isEmpty()) {
- NodeState nodeState = NodeState.valueOf(filterState.toUpperCase());
- switch (nodeState) {
- case DECOMMISSIONED:
- case LOST:
- case REBOOTED:
- rmNodes = this.rm.getRMContext().getInactiveRMNodes().values();
- isInactive = true;
- break;
+
+ NodeState acceptedState = null;
+ boolean all = false;
+
+ if (state != null && !state.isEmpty()) {
+ if (state.equalsIgnoreCase("all")) {
+ all = true;
+ } else {
+ acceptedState = NodeState.valueOf(state.toUpperCase());
}
}
+
+ // getRMNodes() contains nodes that are NEW, RUNNING OR UNHEALTHY
NodesInfo allNodes = new NodesInfo();
- for (RMNode ni : rmNodes) {
- NodeInfo nodeInfo = new NodeInfo(ni, sched);
- if (filterState != null) {
- if (!(nodeInfo.getState().equalsIgnoreCase(filterState))) {
- continue;
- }
- } else {
- // No filter. User is asking for all nodes. Make sure you skip the
- // unhealthy nodes.
- if (ni.getState() == NodeState.UNHEALTHY) {
- continue;
- }
- }
- if ((healthState != null) && (!healthState.isEmpty())) {
- LOG.info("heatlh state is : " + healthState);
- if (!healthState.equalsIgnoreCase("true")
- && !healthState.equalsIgnoreCase("false")) {
- String msg = "Error: You must specify either true or false to query on health";
- throw new BadRequestException(msg);
+ for (RMNode ni : this.rm.getRMContext().getRMNodes().values()) {
+ if (all || (acceptedState == null && ni.getState() != NodeState.UNHEALTHY)
+ || acceptedState == ni.getState()) {
+ NodeInfo nodeInfo = new NodeInfo(ni, sched);
+ allNodes.add(nodeInfo);
+ }
+ }
+
+ // getInactiveNodes() contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
+ if (all || (acceptedState != null &&
+ (acceptedState == NodeState.DECOMMISSIONED ||
+ acceptedState == NodeState.LOST ||
+ acceptedState == NodeState.REBOOTED))) {
+ for (RMNode ni : this.rm.getRMContext().getInactiveRMNodes().values()) {
+ if (all || acceptedState == ni.getState()) {
+ NodeInfo nodeInfo = new NodeInfo(ni, sched);
+ nodeInfo.setNodeHTTPAddress(EMPTY);
+ allNodes.add(nodeInfo);
}
- if ((ni.getState() != NodeState.UNHEALTHY)
- != Boolean.parseBoolean(healthState)) {
- continue;
- }
- }
- if (isInactive) {
- nodeInfo.setNodeHTTPAddress(EMPTY);
}
- allNodes.add(nodeInfo);
}
+
return allNodes;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1490892&r1=1490891&r2=1490892&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Sat Jun 8 00:55:25 2013
@@ -163,7 +163,7 @@ public class TestRMWebServicesNodes exte
}
@Test
- public void testNodesQueryState() throws JSONException, Exception {
+ public void testNodesQueryNew() throws JSONException, Exception {
WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121);
@@ -172,7 +172,7 @@ public class TestRMWebServicesNodes exte
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("nodes").queryParam("state", NodeState.RUNNING.toString())
+ .path("nodes").queryParam("state", NodeState.NEW.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@@ -184,7 +184,7 @@ public class TestRMWebServicesNodes exte
assertEquals("incorrect number of elements", 1, nodeArray.length());
JSONObject info = nodeArray.getJSONObject(0);
- verifyNodeInfo(info, nm1);
+ verifyNodeInfo(info, nm2);
}
@Test
@@ -306,7 +306,7 @@ public class TestRMWebServicesNodes exte
}
@Test
- public void testNodesQueryHealthy() throws JSONException, Exception {
+ public void testNodesQueryRunning() throws JSONException, Exception {
WebResource r = resource();
MockNM nm1 = rm.registerNode("h1:1234", 5120);
MockNM nm2 = rm.registerNode("h2:1235", 5121);
@@ -314,7 +314,7 @@ public class TestRMWebServicesNodes exte
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("nodes").queryParam("healthy", "true")
+ .path("nodes").queryParam("state", "running")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
@@ -322,54 +322,7 @@ public class TestRMWebServicesNodes exte
JSONObject nodes = json.getJSONObject("nodes");
assertEquals("incorrect number of elements", 1, nodes.length());
JSONArray nodeArray = nodes.getJSONArray("node");
- assertEquals("incorrect number of elements", 2, nodeArray.length());
- }
-
- @Test
- public void testNodesQueryHealthyCase() throws JSONException, Exception {
- WebResource r = resource();
- MockNM nm1 = rm.registerNode("h1:1234", 5120);
- MockNM nm2 = rm.registerNode("h2:1235", 5121);
- rm.sendNodeStarted(nm1);
- rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
- rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("nodes").queryParam("healthy", "TRUe")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- JSONObject json = response.getEntity(JSONObject.class);
- assertEquals("incorrect number of elements", 1, json.length());
- JSONObject nodes = json.getJSONObject("nodes");
- assertEquals("incorrect number of elements", 1, nodes.length());
- JSONArray nodeArray = nodes.getJSONArray("node");
- assertEquals("incorrect number of elements", 2, nodeArray.length());
-
- }
-
- @Test
- public void testNodesQueryHealthyAndState() throws JSONException, Exception {
- WebResource r = resource();
- MockNM nm1 = rm.registerNode("h1:1234", 5120);
- MockNM nm2 = rm.registerNode("h2:1235", 5121);
- rm.sendNodeStarted(nm1);
- rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
- rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
- RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
- .get(nm1.getNodeId());
- NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(false,
- "test health report", System.currentTimeMillis());
- node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth,
- new ArrayList<ContainerStatus>(), null, null));
- rm.NMwaitForState(nm1.getNodeId(), NodeState.UNHEALTHY);
-
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("nodes").queryParam("healthy", "true")
- .queryParam("state", NodeState.RUNNING.toString())
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- JSONObject json = response.getEntity(JSONObject.class);
- assertEquals("incorrect number of elements", 1, json.length());
- assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes"));
+ assertEquals("incorrect number of elements", 1, nodeArray.length());
}
@Test
@@ -381,7 +334,7 @@ public class TestRMWebServicesNodes exte
rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("nodes").queryParam("healthy", "false")
+ .path("nodes").queryParam("state", "UNHEALTHY")
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
@@ -389,42 +342,6 @@ public class TestRMWebServicesNodes exte
assertEquals("nodes is not null", JSONObject.NULL, json.get("nodes"));
}
- @Test
- public void testNodesQueryHealthyInvalid() throws JSONException, Exception {
- WebResource r = resource();
- rm.registerNode("h1:1234", 5120);
- rm.registerNode("h2:1235", 5121);
-
- try {
- r.path("ws").path("v1").path("cluster").path("nodes")
- .queryParam("healthy", "tr").accept(MediaType.APPLICATION_JSON)
- .get(JSONObject.class);
- fail("should have thrown exception querying invalid healthy string");
- } catch (UniformInterfaceException ue) {
- ClientResponse response = ue.getResponse();
- assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
- assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
- JSONObject msg = response.getEntity(JSONObject.class);
- JSONObject exception = msg.getJSONObject("RemoteException");
- assertEquals("incorrect number of elements", 3, exception.length());
- String message = exception.getString("message");
- String type = exception.getString("exception");
- String classname = exception.getString("javaClassName");
- WebServicesTestUtils
- .checkStringMatch(
- "exception message",
- "java.lang.Exception: Error: You must specify either true or false to query on health",
- message);
- WebServicesTestUtils.checkStringMatch("exception type",
- "BadRequestException", type);
- WebServicesTestUtils.checkStringMatch("exception classname",
- "org.apache.hadoop.yarn.webapp.BadRequestException", classname);
-
- } finally {
- rm.stop();
- }
- }
-
public void testNodesHelper(String path, String media) throws JSONException,
Exception {
WebResource r = resource();
@@ -689,6 +606,30 @@ public class TestRMWebServicesNodes exte
assertEquals("incorrect number of elements", 2, nodes.getLength());
rm.stop();
}
+
+ @Test
+ public void testQueryAll() throws Exception {
+ WebResource r = resource();
+ MockNM nm1 = rm.registerNode("h1:1234", 5120);
+ MockNM nm2 = rm.registerNode("h2:1235", 5121);
+ MockNM nm3 = rm.registerNode("h3:1236", 5122);
+ rm.sendNodeStarted(nm1);
+ rm.sendNodeStarted(nm3);
+ rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+ rm.NMwaitForState(nm2.getNodeId(), NodeState.NEW);
+ rm.sendNodeLost(nm3);
+
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("nodes").queryParam("state", "aLl")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ JSONObject nodes = json.getJSONObject("nodes");
+ assertEquals("incorrect number of elements", 1, nodes.length());
+ JSONArray nodeArray = nodes.getJSONArray("node");
+ assertEquals("incorrect number of elements", 3, nodeArray.length());
+ }
public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
Exception {