You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2009/12/07 07:12:41 UTC
svn commit: r887844 - in /hadoop/mapreduce/trunk: ./
src/java/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/mapreduce/jobhistory/ src/test/
src/test/mapred/org/apache/hadoop/mapred/ src/webapps/job/
Author: sharad
Date: Mon Dec 7 06:12:40 2009
New Revision: 887844
URL: http://svn.apache.org/viewvc?rev=887844&view=rev
Log:
MAPREDUCE-1185. Redirect running job url to history url if job is already retired. Contributed by Amareshwari Sriramadasu and Sharad Agarwal.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java
hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=887844&r1=887843&r2=887844&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Dec 7 06:12:40 2009
@@ -60,6 +60,9 @@
MAPREDUCE-1119. When tasks fail to report status, show tasks's stack dump
before killing. (Aaron Kimball via tomwhite)
+ MAPREDUCE-1185. Redirect running job url to history url if job is already
+ retired. (Amareshwari Sriramadasu and Sharad Agarwal via sharad)
+
OPTIMIZATIONS
MAPREDUCE-270. Fix the tasktracker to optionally send an out-of-band
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=887844&r1=887843&r2=887844&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Dec 7 06:12:40 2009
@@ -454,19 +454,10 @@
if (job != null) {
JobStatus status = job.getStatus();
- //set the historyfile and update the tracking url
- String trackingUrl = "";
+ //set the historyfile
if (historyFile != null) {
status.setHistoryFile(historyFile);
- try {
- trackingUrl = "http://" + getJobTrackerMachine() + ":" +
- getInfoPort() + "/jobdetailshistory.jsp?jobid=" +
- jobid + "&logFile=" + URLEncoder.encode(historyFile, "UTF-8");
- } catch (UnsupportedEncodingException e) {
- LOG.warn("Could not create trackingUrl", e);
- }
}
- status.setTrackingUrl(trackingUrl);
// clean up job files from the local disk
job.cleanupLocalizedJobConf(job.getProfile().getJobID());
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=887844&r1=887843&r2=887844&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Mon Dec 7 06:12:40 2009
@@ -24,8 +24,11 @@
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@@ -80,6 +83,18 @@
private HistoryCleaner historyCleanerThread = null;
+ private Map<JobID, MovedFileInfo> jobHistoryFileMap =
+ Collections.<JobID,MovedFileInfo>synchronizedMap(
+ new LinkedHashMap<JobID, MovedFileInfo>());
+
+ private static class MovedFileInfo {
+ private final String historyFile;
+ private final long timestamp;
+ public MovedFileInfo(String historyFile, long timestamp) {
+ this.historyFile = historyFile;
+ this.timestamp = timestamp;
+ }
+ }
/**
* Initialize Job History Module
* @param jt Job Tracker handle
@@ -196,6 +211,16 @@
}
/**
+ * Given the job id, return the history file path from the cache
+ */
+ public String getHistoryFilePath(JobID jobId) {
+ MovedFileInfo info = jobHistoryFileMap.get(jobId);
+ if (info == null) {
+ return null;
+ }
+ return info.historyFile;
+ }
+ /**
* Create an event writer for the Job represented by the jobID.
* This should be the first call to history for a job
* @param jobId
@@ -383,7 +408,8 @@
historyFileDonePath = new Path(done,
historyFile.getName()).toString();
}
-
+ jobHistoryFileMap.put(id, new MovedFileInfo(historyFileDonePath,
+ System.currentTimeMillis()));
jobTracker.retireJob(org.apache.hadoop.mapred.JobID.downgrade(id),
historyFileDonePath);
@@ -481,6 +507,21 @@
}
}
}
+ //walking over the map to purge entries from jobHistoryFileMap
+ synchronized (jobHistoryFileMap) {
+ Iterator<Entry<JobID, MovedFileInfo>> it =
+ jobHistoryFileMap.entrySet().iterator();
+ while (it.hasNext()) {
+ MovedFileInfo info = it.next().getValue();
+ if (now - info.timestamp > maxAgeOfHistoryFiles) {
+ it.remove();
+ } else {
+ //since entries are in sorted timestamp order, no more entries
+ //are required to be checked
+ break;
+ }
+ }
+ }
} catch (IOException ie) {
LOG.info("Error cleaning up history directory" +
StringUtils.stringifyException(ie));
Modified: hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml?rev=887844&r1=887843&r2=887844&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/mapreduce/trunk/src/test/findbugsExcludeFile.xml Mon Dec 7 06:12:40 2009
@@ -43,6 +43,10 @@
<Class name="org.apache.hadoop.mapred.taskdetails_jsp" />
<Bug code="HRS" />
</Match>
+ <Match>
+ <Class name="org.apache.hadoop.mapred.jobdetails_jsp"/>
+ <Bug pattern="HRS_REQUEST_PARAMETER_TO_HTTP_HEADER"/>
+ </Match>
<!--
Ignore warnings where child class has the same name as
super class. Classes based on Old API shadow names from
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java?rev=887844&r1=887843&r2=887844&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestJobRetire.java Mon Dec 7 06:12:40 2009
@@ -20,6 +20,8 @@
import java.io.File;
import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
import junit.framework.TestCase;
@@ -86,6 +88,20 @@
File file = new File(name);
assertFalse("JobConf file not deleted", file.exists());
+
+ //test redirection
+ URL jobUrl = new URL(rj.getTrackingURL());
+ HttpURLConnection conn = (HttpURLConnection) jobUrl.openConnection();
+ conn.setInstanceFollowRedirects(false);
+ conn.connect();
+ assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, conn.getResponseCode());
+ conn.disconnect();
+
+ URL redirectedUrl = new URL(conn.getHeaderField("Location"));
+ conn = (HttpURLConnection) redirectedUrl.openConnection();
+ conn.connect();
+ assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ conn.disconnect();
return id;
}
Modified: hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp?rev=887844&r1=887843&r2=887844&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/jobdetails.jsp Mon Dec 7 06:12:40 2009
@@ -27,6 +27,8 @@
import="java.text.DecimalFormat"
import="org.apache.hadoop.mapred.*"
import="org.apache.hadoop.util.*"
+ import="org.apache.hadoop.fs.Path"
+ import="org.apache.hadoop.mapreduce.jobhistory.JobHistory"
%>
<%! private static final long serialVersionUID = 1L;
@@ -219,7 +221,14 @@
<%
if (job == null) {
- out.print("<b>Job " + jobId + " not found.</b><br>\n");
+ String historyFile = tracker.getJobHistory().getHistoryFilePath(jobIdObj);
+ if (historyFile == null) {
+ out.println("<h2>Job " + jobId + " not known!</h2>");
+ return;
+ }
+ String historyUrl = "/jobdetailshistory.jsp?jobid=" +
+ jobId + "&logFile=" + historyFile;
+ response.sendRedirect(response.encodeRedirectURL(historyUrl));
return;
}
JobProfile profile = job.getProfile();