You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 04:37:08 UTC
svn commit: r1077066 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
mapred/org/apache/hadoop/mapred/JobHistory.java test/findbugsExcludeFile.xml
test/org/apache/hadoop/mapred/TestJobRetire.java webapps/job/jobdetails.jsp
Author: omalley
Date: Fri Mar 4 03:37:07 2011
New Revision: 1077066
URL: http://svn.apache.org/viewvc?rev=1077066&view=rev
Log:
commit d43124eb4216ad175cfd9312c98736fb91ca51a8
Author: Hemanth Yamijala <yh...@yahoo-inc.com>
Date: Tue Dec 8 11:27:55 2009 +0530
MAPREDUCE:1185 from https://issues.apache.org/jira/secure/attachment/12426630/patch-1185-3-ydist.txt
+++ b/YAHOO-CHANGES.txt
+ MAPREDUCE-1185. Redirect running job url to history url if job is already
+ retired. (Amareshwari Sriramadasu and Sharad Agarwal via sharad)
+
Added:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml
hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1077066&r1=1077065&r2=1077066&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobHistory.java Fri Mar 4 03:37:07 2011
@@ -29,10 +29,14 @@ import java.io.UnsupportedEncodingExcept
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
+import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
@@ -119,6 +123,30 @@ public class JobHistory {
}
};
+ private static Map<JobID, MovedFileInfo> jobHistoryFileMap =
+ Collections.<JobID,MovedFileInfo>synchronizedMap(
+ new LinkedHashMap<JobID, MovedFileInfo>());
+
+ private static class MovedFileInfo {
+ private final String historyFile;
+ private final long timestamp;
+ public MovedFileInfo(String historyFile, long timestamp) {
+ this.historyFile = historyFile;
+ this.timestamp = timestamp;
+ }
+ }
+
+ /**
+ * Given the job id, return the history file path from the cache
+ */
+ public static String getHistoryFilePath(JobID jobId) {
+ MovedFileInfo info = jobHistoryFileMap.get(jobId);
+ if (info == null) {
+ return null;
+ }
+ return info.historyFile;
+ }
+
/**
* A class that manages all the files related to a job. For now
* - writers : list of open files
@@ -238,6 +266,9 @@ public class JobHistory {
historyFileDonePath = new Path(DONE,
historyFile.getName()).toString();
}
+
+ jobHistoryFileMap.put(id, new MovedFileInfo(historyFileDonePath,
+ System.currentTimeMillis()));
jobTracker.historyFileCopied(id, historyFileDonePath);
//purge the job from the cache
@@ -2036,6 +2067,22 @@ public class JobHistory {
}
}
}
+
+ //walking over the map to purge entries from jobHistoryFileMap
+ synchronized (jobHistoryFileMap) {
+ Iterator<Entry<JobID, MovedFileInfo>> it =
+ jobHistoryFileMap.entrySet().iterator();
+ while (it.hasNext()) {
+ MovedFileInfo info = it.next().getValue();
+ if (now - info.timestamp > THIRTY_DAYS_IN_MS) {
+ it.remove();
+ } else {
+ //since entries are in sorted timestamp order, no more entries
+ //are required to be checked
+ break;
+ }
+ }
+ }
} catch (IOException ie) {
LOG.info("Error cleaning up history directory" +
StringUtils.stringifyException(ie));
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml?rev=1077066&r1=1077065&r2=1077066&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/findbugsExcludeFile.xml Fri Mar 4 03:37:07 2011
@@ -16,6 +16,10 @@
<Bug pattern="DLS_DEAD_LOCAL_STORE" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.mapred.jobdetails_jsp"/>
+ <Bug pattern="HRS_REQUEST_PARAMETER_TO_HTTP_HEADER"/>
+ </Match>
+ <Match>
<Field name="_jspx_dependants" />
<Bug pattern="UWF_UNWRITTEN_FIELD" />
</Match>
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java?rev=1077066&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestJobRetire.java Fri Mar 4 03:37:07 2011
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.HttpURLConnection;
+import java.net.URL;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.JobTracker.RetireJobInfo;
+
+/**
+ * Test if the job retire works fine.
+ */
+public class TestJobRetire extends TestCase {
+ static final Path testDir =
+ new Path(System.getProperty("test.build.data","/tmp"),
+ "job-expiry-testing");
+
+ public void testJobRetire() throws Exception {
+ MiniMRCluster mr = null;
+ try {
+ JobConf conf = new JobConf();
+
+ conf.setLong("mapred.job.tracker.retiredjobs.cache.size", 1);
+ conf.setLong("mapred.jobtracker.retirejob.interval", 0);
+ conf.setLong("mapred.jobtracker.retirejob.check", 0);
+ conf.getLong("mapred.jobtracker.completeuserjobs.maximum", 0);
+ mr = new MiniMRCluster(0, 0, 1, "file:///", 1, null, null, null, conf, 0);
+ JobConf jobConf = mr.createJobConf();
+ JobTracker jobtracker = mr.getJobTrackerRunner().getJobTracker();
+
+ Path inDir = new Path(testDir, "input1");
+ Path outDir = new Path(testDir, "output1");
+
+ JobID id1 = validateJobRetire(jobConf, inDir, outDir, jobtracker);
+
+ outDir = new Path(testDir, "output2");
+ JobID id2 = validateJobRetire(jobConf, inDir, outDir, jobtracker);
+
+ assertNull("Job not removed from cache", jobtracker.getJobStatus(id1));
+
+ assertEquals("Total job in cache not correct",
+ 1, jobtracker.getAllJobs().length);
+ } finally {
+ if (mr != null) { mr.shutdown();}
+ }
+ }
+
+ private JobID validateJobRetire(JobConf jobConf, Path inDir, Path outDir,
+ JobTracker jobtracker) throws IOException {
+
+ RunningJob rj = UtilsForTests.runJob(jobConf, inDir, outDir, 0, 0);
+ rj.waitForCompletion();
+ assertTrue(rj.isSuccessful());
+ JobID id = rj.getID();
+
+ JobInProgress job = jobtracker.getJob(id);
+ //wait for job to get retired
+ for (int i = 0; i < 10 && job != null; i++) {
+ UtilsForTests.waitFor(1000);
+ job = jobtracker.getJob(id);
+ }
+ assertNull("Job did not retire", job);
+ RetireJobInfo retired = jobtracker.retireJobs.get(id);
+ assertTrue("History url not set", retired.getHistoryFile() != null &&
+ retired.getHistoryFile().length() > 0);
+ assertNotNull("Job is not in cache", jobtracker.getJobStatus(id));
+
+ // get the job conf filename
+ String name = jobtracker.getLocalJobFilePath(id);
+ File file = new File(name);
+
+ assertFalse("JobConf file not deleted", file.exists());
+ //test redirection
+ URL jobUrl = new URL(rj.getTrackingURL());
+ HttpURLConnection conn = (HttpURLConnection) jobUrl.openConnection();
+ conn.setInstanceFollowRedirects(false);
+ conn.connect();
+ assertEquals(HttpURLConnection.HTTP_MOVED_TEMP, conn.getResponseCode());
+ conn.disconnect();
+
+ URL redirectedUrl = new URL(conn.getHeaderField("Location"));
+ conn = (HttpURLConnection) redirectedUrl.openConnection();
+ conn.connect();
+ assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ conn.disconnect();
+
+ return id;
+ }
+
+}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp?rev=1077066&r1=1077065&r2=1077066&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/webapps/job/jobdetails.jsp Fri Mar 4 03:37:07 2011
@@ -197,7 +197,14 @@
<%
if (job == null) {
- out.print("<b>Job " + jobId + " not found.</b><br>\n");
+ String historyFile = JobHistory.getHistoryFilePath(jobIdObj);
+ if (historyFile == null) {
+ out.println("<h2>Job " + jobId + " not known!</h2>");
+ return;
+ }
+ String historyUrl = "/jobdetailshistory.jsp?jobid=" + jobId +
+ "&logFile=" + JobHistory.JobInfo.encodeJobHistoryFilePath(historyFile);
+ response.sendRedirect(response.encodeRedirectURL(historyUrl));
return;
}
JobProfile profile = job.getProfile();