You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ss...@apache.org on 2012/03/06 20:53:09 UTC
svn commit: r1297673 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/j...
Author: sseth
Date: Tue Mar 6 19:53:09 2012
New Revision: 1297673
URL: http://svn.apache.org/viewvc?rev=1297673&view=rev
Log:
merge MAPREDUCE-3944 from trunk
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java
- copied unchanged from r1297672, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/MockHistoryJobs.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Mar 6 19:53:09 2012
@@ -91,6 +91,10 @@ Release 0.23.2 - UNRELEASED
MAPREDUCE-2855. Passing a cached class-loader to ResourceBundle creator to
minimize counter names lookup time. (Siddarth Seth via vinodkv)
+ MAPREDUCE-3944. Change the history jobs/ webservice to return partial job
+ info for a significant performance improvement. (Robert Joseph Evans via
+ sseth)
+
BUG FIXES
MAPREDUCE-3918 proc_historyserver no longer in command line arguments for
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedJob.java Tue Mar 6 19:53:09 2012
@@ -287,7 +287,7 @@ public class CompletedJob implements org
}
//History data is leisurely loaded when task level data is requested
- private synchronized void loadFullHistoryData(boolean loadTasks,
+ protected synchronized void loadFullHistoryData(boolean loadTasks,
Path historyFileAbsolute) throws IOException {
LOG.info("Loading history file: [" + historyFileAbsolute + "]");
if (this.jobInfo != null) {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsWebServices.java Tue Mar 6 19:53:09 2012
@@ -31,6 +31,7 @@ import javax.ws.rs.core.UriInfo;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.v2.api.records.AMInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.TaskId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
@@ -184,35 +185,32 @@ public class HsWebServices {
break;
}
- // getAllJobs only gives you a partial we want a full
- Job fullJob = appCtx.getJob(job.getID());
- if (fullJob == null) {
- continue;
- }
-
- JobInfo jobInfo = new JobInfo(fullJob);
// can't really validate queue is a valid one since queues could change
if (queueQuery != null && !queueQuery.isEmpty()) {
- if (!jobInfo.getQueueName().equals(queueQuery)) {
+ if (!job.getQueueName().equals(queueQuery)) {
continue;
}
}
if (userQuery != null && !userQuery.isEmpty()) {
- if (!jobInfo.getUserName().equals(userQuery)) {
+ if (!job.getUserName().equals(userQuery)) {
continue;
}
}
+ JobReport report = job.getReport();
+
if (checkStart
- && (jobInfo.getStartTime() < sBegin || jobInfo.getStartTime() > sEnd)) {
+ && (report.getStartTime() < sBegin || report.getStartTime() > sEnd)) {
continue;
}
if (checkEnd
- && (jobInfo.getFinishTime() < fBegin || jobInfo.getFinishTime() > fEnd)) {
+ && (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd)) {
continue;
}
-
+
+ JobInfo jobInfo = new JobInfo(job);
+
allJobs.add(jobInfo);
num++;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/dao/JobInfo.java Tue Mar 6 19:53:09 2012
@@ -55,18 +55,18 @@ public class JobInfo {
protected int mapsCompleted;
protected int reducesTotal;
protected int reducesCompleted;
- protected boolean uberized;
+ protected Boolean uberized;
protected String diagnostics;
- protected long avgMapTime = 0;
- protected long avgReduceTime = 0;
- protected long avgShuffleTime = 0;
- protected long avgMergeTime = 0;
- protected int failedReduceAttempts = 0;
- protected int killedReduceAttempts = 0;
- protected int successfulReduceAttempts = 0;
- protected int failedMapAttempts = 0;
- protected int killedMapAttempts = 0;
- protected int successfulMapAttempts = 0;
+ protected Long avgMapTime;
+ protected Long avgReduceTime;
+ protected Long avgShuffleTime;
+ protected Long avgMergeTime;
+ protected Integer failedReduceAttempts;
+ protected Integer killedReduceAttempts;
+ protected Integer successfulReduceAttempts;
+ protected Integer failedMapAttempts;
+ protected Integer killedMapAttempts;
+ protected Integer successfulMapAttempts;
protected ArrayList<ConfEntryInfo> acls;
@XmlTransient
@@ -80,7 +80,7 @@ public class JobInfo {
public JobInfo(Job job) {
this.id = MRApps.toString(job.getID());
JobReport report = job.getReport();
- countTasksAndAttempts(job);
+
this.mapsTotal = job.getTotalMaps();
this.mapsCompleted = job.getCompletedMaps();
this.reducesTotal = job.getTotalReduces();
@@ -91,19 +91,33 @@ public class JobInfo {
this.queue = job.getQueueName();
this.user = job.getUserName();
this.state = job.getState().toString();
- this.uberized = job.isUber();
- this.diagnostics = "";
- List<String> diagnostics = job.getDiagnostics();
- if (diagnostics != null && !diagnostics.isEmpty()) {
- StringBuffer b = new StringBuffer();
- for (String diag : diagnostics) {
- b.append(diag);
- }
- this.diagnostics = b.toString();
- }
this.acls = new ArrayList<ConfEntryInfo>();
+
if (job instanceof CompletedJob) {
+ avgMapTime = 0l;
+ avgReduceTime = 0l;
+ avgShuffleTime = 0l;
+ avgMergeTime = 0l;
+ failedReduceAttempts = 0;
+ killedReduceAttempts = 0;
+ successfulReduceAttempts = 0;
+ failedMapAttempts = 0;
+ killedMapAttempts = 0;
+ successfulMapAttempts = 0;
+ countTasksAndAttempts(job);
+ this.uberized = job.isUber();
+ this.diagnostics = "";
+ List<String> diagnostics = job.getDiagnostics();
+ if (diagnostics != null && !diagnostics.isEmpty()) {
+ StringBuffer b = new StringBuffer();
+ for (String diag : diagnostics) {
+ b.append(diag);
+ }
+ this.diagnostics = b.toString();
+ }
+
+
Map<JobACL, AccessControlList> allacls = job.getJobACLs();
if (allacls != null) {
for (Map.Entry<JobACL, AccessControlList> entry : allacls.entrySet()) {
@@ -122,43 +136,43 @@ public class JobInfo {
return numReduces;
}
- public long getAvgMapTime() {
+ public Long getAvgMapTime() {
return avgMapTime;
}
- public long getAvgReduceTime() {
+ public Long getAvgReduceTime() {
return avgReduceTime;
}
- public long getAvgShuffleTime() {
+ public Long getAvgShuffleTime() {
return avgShuffleTime;
}
- public long getAvgMergeTime() {
+ public Long getAvgMergeTime() {
return avgMergeTime;
}
- public long getFailedReduceAttempts() {
+ public Integer getFailedReduceAttempts() {
return failedReduceAttempts;
}
- public long getKilledReduceAttempts() {
+ public Integer getKilledReduceAttempts() {
return killedReduceAttempts;
}
- public long getSuccessfulReduceAttempts() {
+ public Integer getSuccessfulReduceAttempts() {
return successfulReduceAttempts;
}
- public long getFailedMapAttempts() {
+ public Integer getFailedMapAttempts() {
return failedMapAttempts;
}
- public long getKilledMapAttempts() {
+ public Integer getKilledMapAttempts() {
return killedMapAttempts;
}
- public long getSuccessfulMapAttempts() {
+ public Integer getSuccessfulMapAttempts() {
return successfulMapAttempts;
}
@@ -210,7 +224,7 @@ public class JobInfo {
return this.finishTime;
}
- public boolean isUber() {
+ public Boolean isUber() {
return this.uberized;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobs.java Tue Mar 6 19:53:09 2012
@@ -27,6 +27,7 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.io.StringReader;
import java.util.Map;
@@ -40,8 +41,11 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs;
+import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs.JobsPair;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -89,13 +93,21 @@ public class TestHsWebServicesJobs exten
final ApplicationAttemptId appAttemptID;
final ApplicationId appID;
final String user = MockJobs.newUserName();
- final Map<JobId, Job> jobs;
+ final Map<JobId, Job> partialJobs;
+ final Map<JobId, Job> fullJobs;
final long startTime = System.currentTimeMillis();
TestAppContext(int appid, int numJobs, int numTasks, int numAttempts) {
appID = MockJobs.newAppID(appid);
appAttemptID = MockJobs.newAppAttemptID(appID, 0);
- jobs = MockJobs.newJobs(appID, numJobs, numTasks, numAttempts);
+ JobsPair jobs;
+ try {
+ jobs = MockHistoryJobs.newHistoryJobs(appID, numJobs, numTasks, numAttempts);
+ } catch (IOException e) {
+ throw new YarnException(e);
+ }
+ partialJobs = jobs.partial;
+ fullJobs = jobs.full;
}
TestAppContext() {
@@ -119,12 +131,16 @@ public class TestHsWebServicesJobs exten
@Override
public Job getJob(JobId jobID) {
- return jobs.get(jobID);
+ return fullJobs.get(jobID);
}
+ public Job getPartialJob(JobId jobID) {
+ return partialJobs.get(jobID);
+ }
+
@Override
public Map<JobId, Job> getAllJobs() {
- return jobs; // OK
+ return partialJobs; // OK
}
@SuppressWarnings("rawtypes")
@@ -204,8 +220,8 @@ public class TestHsWebServicesJobs exten
JSONArray arr = jobs.getJSONArray("job");
assertEquals("incorrect number of elements", 1, arr.length());
JSONObject info = arr.getJSONObject(0);
- Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
- VerifyJobsUtils.verifyHsJob(info, job);
+ Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
+ VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@@ -222,8 +238,8 @@ public class TestHsWebServicesJobs exten
JSONArray arr = jobs.getJSONArray("job");
assertEquals("incorrect number of elements", 1, arr.length());
JSONObject info = arr.getJSONObject(0);
- Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
- VerifyJobsUtils.verifyHsJob(info, job);
+ Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
+ VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@@ -239,8 +255,8 @@ public class TestHsWebServicesJobs exten
JSONArray arr = jobs.getJSONArray("job");
assertEquals("incorrect number of elements", 1, arr.length());
JSONObject info = arr.getJSONObject(0);
- Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
- VerifyJobsUtils.verifyHsJob(info, job);
+ Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
+ VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@@ -261,10 +277,35 @@ public class TestHsWebServicesJobs exten
assertEquals("incorrect number of elements", 1, jobs.getLength());
NodeList job = dom.getElementsByTagName("job");
assertEquals("incorrect number of elements", 1, job.getLength());
- verifyHsJobXML(job, appContext);
-
+ verifyHsJobPartialXML(job, appContext);
}
+ public void verifyHsJobPartialXML(NodeList nodes, TestAppContext appContext) {
+
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+
+ for (int i = 0; i < nodes.getLength(); i++) {
+ Element element = (Element) nodes.item(i);
+
+ Job job = appContext.getPartialJob(MRApps.toJobID(WebServicesTestUtils
+ .getXmlString(element, "id")));
+ assertNotNull("Job not found - output incorrect", job);
+
+ VerifyJobsUtils.verifyHsJobGeneric(job,
+ WebServicesTestUtils.getXmlString(element, "id"),
+ WebServicesTestUtils.getXmlString(element, "user"),
+ WebServicesTestUtils.getXmlString(element, "name"),
+ WebServicesTestUtils.getXmlString(element, "state"),
+ WebServicesTestUtils.getXmlString(element, "queue"),
+ WebServicesTestUtils.getXmlLong(element, "startTime"),
+ WebServicesTestUtils.getXmlLong(element, "finishTime"),
+ WebServicesTestUtils.getXmlInt(element, "mapsTotal"),
+ WebServicesTestUtils.getXmlInt(element, "mapsCompleted"),
+ WebServicesTestUtils.getXmlInt(element, "reducesTotal"),
+ WebServicesTestUtils.getXmlInt(element, "reducesCompleted"));
+ }
+ }
+
public void verifyHsJobXML(NodeList nodes, TestAppContext appContext) {
assertEquals("incorrect number of elements", 1, nodes.getLength());
@@ -320,7 +361,7 @@ public class TestHsWebServicesJobs exten
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("job");
- VerifyJobsUtils.verifyHsJob(info, jobsMap.get(id));
+ VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id));
}
}
@@ -356,7 +397,7 @@ public class TestHsWebServicesJobs exten
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("job");
- VerifyJobsUtils.verifyHsJob(info, jobsMap.get(id));
+ VerifyJobsUtils.verifyHsJob(info, appContext.getJob(id));
}
}
@@ -694,7 +735,7 @@ public class TestHsWebServicesJobs exten
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("jobAttempts");
- verifyHsJobAttempts(info, jobsMap.get(id));
+ verifyHsJobAttempts(info, appContext.getJob(id));
}
}
@@ -712,7 +753,7 @@ public class TestHsWebServicesJobs exten
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("jobAttempts");
- verifyHsJobAttempts(info, jobsMap.get(id));
+ verifyHsJobAttempts(info, appContext.getJob(id));
}
}
@@ -730,7 +771,7 @@ public class TestHsWebServicesJobs exten
JSONObject json = response.getEntity(JSONObject.class);
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("jobAttempts");
- verifyHsJobAttempts(info, jobsMap.get(id));
+ verifyHsJobAttempts(info, appContext.getJob(id));
}
}
@@ -754,7 +795,7 @@ public class TestHsWebServicesJobs exten
NodeList attempts = dom.getElementsByTagName("jobAttempts");
assertEquals("incorrect number of elements", 1, attempts.getLength());
NodeList info = dom.getElementsByTagName("jobAttempt");
- verifyHsJobAttemptsXML(info, jobsMap.get(id));
+ verifyHsJobAttemptsXML(info, appContext.getJob(id));
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/TestHsWebServicesJobsQuery.java Tue Mar 6 19:53:09 2012
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTru
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Map;
@@ -34,8 +35,11 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.MockJobs;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
+import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs;
+import org.apache.hadoop.mapreduce.v2.hs.MockHistoryJobs.JobsPair;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.yarn.Clock;
+import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -73,11 +77,19 @@ public class TestHsWebServicesJobsQuery
static class TestAppContext implements AppContext {
final String user = MockJobs.newUserName();
- final Map<JobId, Job> jobs;
+ final Map<JobId, Job> fullJobs;
+ final Map<JobId, Job> partialJobs;
final long startTime = System.currentTimeMillis();
TestAppContext(int numJobs, int numTasks, int numAttempts) {
- jobs = MockJobs.newJobs(numJobs, numTasks, numAttempts);
+ JobsPair jobs;
+ try {
+ jobs = MockHistoryJobs.newHistoryJobs(numJobs, numTasks, numAttempts);
+ } catch (IOException e) {
+ throw new YarnException(e);
+ }
+ partialJobs = jobs.partial;
+ fullJobs = jobs.full;
}
TestAppContext() {
@@ -101,12 +113,16 @@ public class TestHsWebServicesJobsQuery
@Override
public Job getJob(JobId jobID) {
- return jobs.get(jobID);
+ return fullJobs.get(jobID);
}
+ public Job getPartialJob(JobId jobID) {
+ return partialJobs.get(jobID);
+ }
+
@Override
public Map<JobId, Job> getAllJobs() {
- return jobs; // OK
+ return partialJobs; // OK
}
@SuppressWarnings("rawtypes")
@@ -199,8 +215,8 @@ public class TestHsWebServicesJobsQuery
assertEquals("incorrect number of elements", 3, arr.length());
// just verify one of them.
JSONObject info = arr.getJSONObject(0);
- Job job = appContext.getJob(MRApps.toJobID(info.getString("id")));
- VerifyJobsUtils.verifyHsJob(info, job);
+ Job job = appContext.getPartialJob(MRApps.toJobID(info.getString("id")));
+ VerifyJobsUtils.verifyHsJobPartial(info, job);
}
@Test
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/webapp/VerifyJobsUtils.java Tue Mar 6 19:53:09 2012
@@ -32,11 +32,20 @@ import org.codehaus.jettison.json.JSONOb
public class VerifyJobsUtils {
- public static void verifyHsJob(JSONObject info, Job job) throws JSONException {
+ public static void verifyHsJobPartial(JSONObject info, Job job) throws JSONException {
+ assertEquals("incorrect number of elements", 11, info.length());
- // this is 23 instead of 24 because acls not being checked since
- // we are using mock job instead of CompletedJob
- assertEquals("incorrect number of elements", 23, info.length());
+ // everyone access fields
+ verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
+ info.getString("name"), info.getString("state"),
+ info.getString("queue"), info.getLong("startTime"),
+ info.getLong("finishTime"), info.getInt("mapsTotal"),
+ info.getInt("mapsCompleted"), info.getInt("reducesTotal"),
+ info.getInt("reducesCompleted"));
+ }
+
+ public static void verifyHsJob(JSONObject info, Job job) throws JSONException {
+ assertEquals("incorrect number of elements", 24, info.length());
// everyone access fields
verifyHsJobGeneric(job, info.getString("id"), info.getString("user"),
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm?rev=1297673&r1=1297672&r2=1297673&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm Tue Mar 6 19:53:09 2012
@@ -129,7 +129,7 @@ History Server REST API's.
** Jobs API
- The jobs resource provides a list of the MapReduce jobs that have finished.
+ The jobs resource provides a list of the MapReduce jobs that have finished. It does not currently return a full list of parameters
*** URI
@@ -160,7 +160,9 @@ History Server REST API's.
*** Elements of the <jobs> object
When you make a request for the list of jobs, the information will be returned as an array of job objects.
- See also {{Job API}} for syntax of the job object.
+ See also {{Job API}} for syntax of the job object. Except this is a subset of a full job. Only startTime,
+ finishTime, id, name, queue, user, state, mapsTotal, mapsCompleted, reducesTotal, and reducesCompleted are
+ returned.
*---------------+--------------+-------------------------------+
|| Item || Data Type || Description |
@@ -194,73 +196,29 @@ History Server REST API's.
"jobs" : {
"job" : [
{
- "avgReduceTime" : 833,
- "failedReduceAttempts" : 0,
"state" : "SUCCEEDED",
- "successfulReduceAttempts" : 1,
- "acls" : [
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-modify-job"
- },
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-view-job"
- }
- ],
"user" : "user1",
"reducesTotal" : 1,
"mapsCompleted" : 1,
"startTime" : 1326381344489,
"id" : "job_1326381300833_1_1",
- "avgMapTime" : 2671,
- "successfulMapAttempts" : 1,
"name" : "word count",
- "avgShuffleTime" : 2540,
"reducesCompleted" : 1,
- "diagnostics" : "",
- "failedMapAttempts" : 0,
- "avgMergeTime" : 2570,
- "killedReduceAttempts" : 0,
"mapsTotal" : 1,
"queue" : "default",
- "uberized" : false,
- "killedMapAttempts" : 0,
"finishTime" : 1326381356010
},
{
- "avgReduceTime" : 124961,
- "failedReduceAttempts" : 0,
"state" : "SUCCEEDED",
- "successfulReduceAttempts" : 1,
- "acls" : [
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-modify-job"
- },
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-view-job"
- }
- ],
"user" : "user1",
"reducesTotal" : 1,
"mapsCompleted" : 1,
"startTime" : 1326381446529,
"id" : "job_1326381300833_2_2",
- "avgMapTime" : 2638,
- "successfulMapAttempts" : 1,
"name" : "Sleep job",
- "avgShuffleTime" : 2540,
"reducesCompleted" : 1,
- "diagnostics" : "",
- "failedMapAttempts" : 0,
- "avgMergeTime" : 2589,
- "killedReduceAttempts" : 0,
"mapsTotal" : 1,
"queue" : "default",
- "uberized" : false,
- "killedMapAttempts" : 0,
"finishTime" : 1326381582106
}
]
@@ -303,26 +261,6 @@ History Server REST API's.
<mapsCompleted>1</mapsCompleted>
<reducesTotal>1</reducesTotal>
<reducesCompleted>1</reducesCompleted>
- <uberized>false</uberized>
- <diagnostics/>
- <avgMapTime>2671</avgMapTime>
- <avgReduceTime>833</avgReduceTime>
- <avgShuffleTime>2540</avgShuffleTime>
- <avgMergeTime>2570</avgMergeTime>
- <failedReduceAttempts>0</failedReduceAttempts>
- <killedReduceAttempts>0</killedReduceAttempts>
- <successfulReduceAttempts>1</successfulReduceAttempts>
- <failedMapAttempts>0</failedMapAttempts>
- <killedMapAttempts>0</killedMapAttempts>
- <successfulMapAttempts>1</successfulMapAttempts>
- <acls>
- <name>mapreduce.job.acl-modify-job</name>
- <value> </value>
- </acls>
- <acls>
- <name>mapreduce.job.acl-view-job</name>
- <value> </value>
- </acls>
</job>
<job>
<startTime>1326381446529</startTime>
@@ -336,26 +274,6 @@ History Server REST API's.
<mapsCompleted>1</mapsCompleted>
<reducesTotal>1</reducesTotal>
<reducesCompleted>1</reducesCompleted>
- <uberized>false</uberized>
- <diagnostics/>
- <avgMapTime>2638</avgMapTime>
- <avgReduceTime>124961</avgReduceTime>
- <avgShuffleTime>2540</avgShuffleTime>
- <avgMergeTime>2589</avgMergeTime>
- <failedReduceAttempts>0</failedReduceAttempts>
- <killedReduceAttempts>0</killedReduceAttempts>
- <successfulReduceAttempts>1</successfulReduceAttempts>
- <failedMapAttempts>0</failedMapAttempts>
- <killedMapAttempts>0</killedMapAttempts>
- <successfulMapAttempts>1</successfulMapAttempts>
- <acls>
- <name>mapreduce.job.acl-modify-job</name>
- <value> </value>
- </acls>
- <acls>
- <name>mapreduce.job.acl-view-job</name>
- <value> </value>
- </acls>
</job>
</jobs>
+---+