You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/04/02 23:09:03 UTC
svn commit: r1308569 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src...
Author: bobby
Date: Mon Apr 2 21:09:03 2012
New Revision: 1308569
URL: http://svn.apache.org/viewvc?rev=1308569&view=rev
Log:
svn merge -c 1308566 from trunk. FIXES MAPREDUCE-4024. RM webservices can't query on finalStatus (Tom Graves via bobby)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Apr 2 21:09:03 2012
@@ -17,6 +17,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4089. Hung Tasks never time out. (Robert Evans via tgraves)
+ MAPREDUCE-4024. RM webservices can't query on finalStatus (Tom Graves
+ via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java Mon Apr 2 21:09:03 2012
@@ -31,6 +31,7 @@ import javax.ws.rs.core.UriInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
@@ -96,6 +97,7 @@ public class HsWebServices {
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public JobsInfo getJobs(@QueryParam("user") String userQuery,
@QueryParam("limit") String count,
+ @QueryParam("state") String stateQuery,
@QueryParam("queue") String queueQuery,
@QueryParam("startedTimeBegin") String startedBegin,
@QueryParam("startedTimeEnd") String startedEnd,
@@ -185,6 +187,13 @@ public class HsWebServices {
break;
}
+ if (stateQuery != null && !stateQuery.isEmpty()) {
+ JobState.valueOf(stateQuery);
+ if (!job.getState().toString().equalsIgnoreCase(stateQuery)) {
+ continue;
+ }
+ }
+
// can't really validate queue is a valid one since queues could change
if (queueQuery != null && !queueQuery.isEmpty()) {
if (!job.getQueueName().equals(queueQuery)) {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java Mon Apr 2 21:09:03 2012
@@ -32,6 +32,7 @@ import javax.ws.rs.core.MediaType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
@@ -120,7 +121,7 @@ public class TestHsWebServicesJobsQuery
public Job getPartialJob(JobId jobID) {
return partialJobs.get(jobID);
}
-
+
@Override
public Map<JobId, Job> getAllJobs() {
return partialJobs; // OK
@@ -196,6 +197,72 @@ public class TestHsWebServicesJobsQuery
}
@Test
+ public void testJobsQueryStateNone() throws JSONException, Exception {
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("history")
+ .path("mapreduce").path("jobs").queryParam("state", JobState.KILL_WAIT.toString())
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ assertEquals("jobs is not null", JSONObject.NULL, json.get("jobs"));
+ }
+
+ @Test
+ public void testJobsQueryState() throws JSONException, Exception {
+ WebResource r = resource();
+ // we only create 3 jobs and it cycles through states so we should have 3 unique states
+ Map<JobId, Job> jobsMap = appContext.getAllJobs();
+ String queryState = "BOGUS";
+ JobId jid = null;
+ for (Map.Entry<JobId, Job> entry : jobsMap.entrySet()) {
+ jid = entry.getValue().getID();
+ queryState = entry.getValue().getState().toString();
+ break;
+ }
+ ClientResponse response = r.path("ws").path("v1").path("history")
+ .path("mapreduce").path("jobs").queryParam("state", queryState)
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject jobs = json.getJSONObject("jobs");
+ JSONArray arr = jobs.getJSONArray("job");
+ assertEquals("incorrect number of elements", 1, arr.length());
+ JSONObject info = arr.getJSONObject(0);
+ Job job = appContext.getPartialJob(jid);
+ VerifyJobsUtils.verifyHsJobPartial(info, job);
+ }
+
+ @Test
+ public void testJobsQueryStateInvalid() throws JSONException, Exception {
+ WebResource r = resource();
+
+ ClientResponse response = r.path("ws").path("v1").path("history")
+ .path("mapreduce").path("jobs").queryParam("state", "InvalidState")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils
+ .checkStringMatch(
+ "exception message",
+ "No enum const class org.apache.hadoop.mapreduce.v2.api.records.JobState.InvalidState",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "IllegalArgumentException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "java.lang.IllegalArgumentException", classname);
+ }
+
+
+ @Test
public void testJobsQueryUserNone() throws JSONException, Exception {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("history")
@@ -215,6 +282,8 @@ public class TestHsWebServicesJobsQuery
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
JSONObject json = response.getEntity(JSONObject.class);
+ System.out.println(json.toString());
+
assertEquals("incorrect number of elements", 1, json.length());
JSONObject jobs = json.getJSONObject("jobs");
JSONArray arr = jobs.getJSONArray("job");
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Mon Apr 2 21:09:03 2012
@@ -36,6 +36,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -225,6 +226,7 @@ public class RMWebServices {
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public AppsInfo getApps(@Context HttpServletRequest hsr,
@QueryParam("state") String stateQuery,
+ @QueryParam("finalStatus") String finalStatusQuery,
@QueryParam("user") String userQuery,
@QueryParam("queue") String queueQuery,
@QueryParam("limit") String count,
@@ -294,19 +296,25 @@ public class RMWebServices {
.getRMApps();
AppsInfo allApps = new AppsInfo();
for (RMApp rmapp : apps.values()) {
+
if (checkCount && num == countNum) {
break;
}
- AppInfo app = new AppInfo(rmapp, hasAccess(rmapp, hsr));
-
if (stateQuery != null && !stateQuery.isEmpty()) {
RMAppState.valueOf(stateQuery);
- if (!app.getState().equalsIgnoreCase(stateQuery)) {
+ if (!rmapp.getState().toString().equalsIgnoreCase(stateQuery)) {
+ continue;
+ }
+ }
+ if (finalStatusQuery != null && !finalStatusQuery.isEmpty()) {
+ FinalApplicationStatus.valueOf(finalStatusQuery);
+ if (!rmapp.getFinalApplicationStatus().toString()
+ .equalsIgnoreCase(finalStatusQuery)) {
continue;
}
}
if (userQuery != null && !userQuery.isEmpty()) {
- if (!app.getUser().equals(userQuery)) {
+ if (!rmapp.getUser().equals(userQuery)) {
continue;
}
}
@@ -321,19 +329,20 @@ public class RMWebServices {
throw new BadRequestException(e.getMessage());
}
}
- if (!app.getQueue().equals(queueQuery)) {
+ if (!rmapp.getQueue().equals(queueQuery)) {
continue;
}
}
if (checkStart
- && (app.getStartTime() < sBegin || app.getStartTime() > sEnd)) {
+ && (rmapp.getStartTime() < sBegin || rmapp.getStartTime() > sEnd)) {
continue;
}
if (checkEnd
- && (app.getFinishTime() < fBegin || app.getFinishTime() > fEnd)) {
+ && (rmapp.getFinishTime() < fBegin || rmapp.getFinishTime() > fEnd)) {
continue;
}
+ AppInfo app = new AppInfo(rmapp, hasAccess(rmapp, hsr));
allApps.add(app);
num++;
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Mon Apr 2 21:09:03 2012
@@ -30,6 +30,7 @@ import javax.xml.parsers.DocumentBuilder
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -281,6 +282,85 @@ public class TestRMWebServicesApps exten
}
@Test
+ public void testAppsQueryFinalStatus() throws JSONException, Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ RMApp app1 = rm.submitApp(1024);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ System.out.println(json.toString());
+ JSONObject apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ JSONArray array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ verifyAppInfo(array.getJSONObject(0), app1);
+ rm.stop();
+ }
+
+ @Test
+ public void testAppsQueryFinalStatusNone() throws JSONException, Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ rm.submitApp(1024);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("finalStatus", FinalApplicationStatus.KILLED.toString())
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ assertEquals("apps is not null", JSONObject.NULL, json.get("apps"));
+ rm.stop();
+ }
+
+ @Test
+ public void testAppsQueryFinalStatusInvalid() throws JSONException, Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ rm.submitApp(1024);
+ amNodeManager.nodeHeartbeat(true);
+ WebResource r = resource();
+
+ try {
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("finalStatus", "INVALID_test")
+ .accept(MediaType.APPLICATION_JSON).get(JSONObject.class);
+ fail("should have thrown exception on invalid state query");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ WebServicesTestUtils
+ .checkStringMatch(
+ "exception message",
+ "No enum const class org.apache.hadoop.yarn.api.records.FinalApplicationStatus.INVALID_test",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "IllegalArgumentException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "java.lang.IllegalArgumentException", classname);
+
+ } finally {
+ rm.stop();
+ }
+ }
+
+ @Test
public void testAppsQueryUser() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm Mon Apr 2 21:09:03 2012
@@ -149,6 +149,7 @@ History Server REST API's.
------
* user - user name
+ * state - the job state
* queue - queue name
* limit - total number of app objects to be returned
* startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1308569&r1=1308568&r2=1308569&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Mon Apr 2 21:09:03 2012
@@ -890,6 +890,7 @@ ResourceManager REST API's.
------
* state - state of the application
+ * finalStatus - the final status of the application - reported by the application itself
* user - user name
* queue - queue name
* limit - total number of app objects to be returned