You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ss...@apache.org on 2012/10/09 05:23:07 UTC
svn commit: r1395850 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/
hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/...
Author: sseth
Date: Tue Oct 9 03:23:07 2012
New Revision: 1395850
URL: http://svn.apache.org/viewvc?rev=1395850&view=rev
Log:
MAPREDUCE-4705. Fix a bug in job history lookup, which makes older jobs inaccessible despite the presence of a valid history file. (Contributed by Jason Lowe)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1395850&r1=1395849&r2=1395850&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Oct 9 03:23:07 2012
@@ -558,6 +558,10 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4554. Job Credentials are not transmitted if security is turned
off (Benoy Antony via bobby)
+ MAPREDUCE-4705. Fix a bug in job history lookup, which makes older jobs
+ inaccessible despite the presence of a valid history file. (Jason Lowe
+ via sseth)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java?rev=1395850&r1=1395849&r2=1395850&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java Tue Oct 9 03:23:07 2012
@@ -62,6 +62,7 @@ import org.apache.hadoop.mapreduce.v2.jo
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.service.AbstractService;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -130,7 +131,7 @@ public class HistoryFileManager extends
}
}
- private static class JobListCache {
+ static class JobListCache {
private ConcurrentSkipListMap<JobId, HistoryFileInfo> cache;
private int maxSize;
private long maxAge;
@@ -239,12 +240,14 @@ public class HistoryFileManager extends
: HistoryInfoState.IN_INTERMEDIATE;
}
- private synchronized boolean isMovePending() {
+ @VisibleForTesting
+ synchronized boolean isMovePending() {
return state == HistoryInfoState.IN_INTERMEDIATE
|| state == HistoryInfoState.MOVE_FAILED;
}
- private synchronized boolean didMoveFail() {
+ @VisibleForTesting
+ synchronized boolean didMoveFail() {
return state == HistoryInfoState.MOVE_FAILED;
}
@@ -365,7 +368,7 @@ public class HistoryFileManager extends
}
private SerialNumberIndex serialNumberIndex = null;
- private JobListCache jobListCache = null;
+ protected JobListCache jobListCache = null;
// Maintains a list of known done subdirectories.
private final Set<Path> existingDoneSubdirs = Collections
@@ -707,8 +710,8 @@ public class HistoryFileManager extends
* @throws IOException
*/
private HistoryFileInfo scanOldDirsForJob(JobId jobId) throws IOException {
- int jobSerialNumber = JobHistoryUtils.jobSerialNumber(jobId);
- String boxedSerialNumber = String.valueOf(jobSerialNumber);
+ String boxedSerialNumber = JobHistoryUtils.serialNumberDirectoryComponent(
+ jobId, serialNumberFormat);
Set<String> dateStringSet = serialNumberIndex.get(boxedSerialNumber);
if (dateStringSet == null) {
return null;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java?rev=1395850&r1=1395849&r2=1395850&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryParsing.java Tue Oct 9 03:23:07 2012
@@ -60,6 +60,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo;
import org.apache.hadoop.mapreduce.v2.hs.TestJobHistoryEvents.MRAppWithHistory;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.net.DNSToSwitchMapping;
@@ -460,6 +461,51 @@ public class TestJobHistoryParsing {
}
}
+ @Test
+ public void testScanningOldDirs() throws Exception {
+ LOG.info("STARTING testScanningOldDirs");
+ try {
+ Configuration conf = new Configuration();
+ conf
+ .setClass(
+ CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+ MyResolver.class, DNSToSwitchMapping.class);
+ RackResolver.init(conf);
+ MRApp app =
+ new MRAppWithHistory(1, 1, true,
+ this.getClass().getName(), true);
+ app.submit(conf);
+ Job job = app.getContext().getAllJobs().values().iterator().next();
+ JobId jobId = job.getID();
+ LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
+ app.waitForState(job, JobState.SUCCEEDED);
+
+ // make sure all events are flushed
+ app.waitForState(Service.STATE.STOPPED);
+
+ HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
+ hfm.init(conf);
+ HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
+ Assert.assertNotNull("Unable to locate job history", fileInfo);
+
+ // force the manager to "forget" the job
+ hfm.deleteJobFromJobListCache(fileInfo);
+ final int msecPerSleep = 10;
+ int msecToSleep = 10 * 1000;
+ while (fileInfo.isMovePending() && msecToSleep > 0) {
+ Assert.assertTrue(!fileInfo.didMoveFail());
+ msecToSleep -= msecPerSleep;
+ Thread.sleep(msecPerSleep);
+ }
+ Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);
+
+ fileInfo = hfm.getFileInfo(jobId);
+ Assert.assertNotNull("Unable to locate old job history", fileInfo);
+ } finally {
+ LOG.info("FINISHED testScanningOldDirs");
+ }
+ }
+
static class MRAppWithHistoryWithFailedAttempt extends MRAppWithHistory {
public MRAppWithHistoryWithFailedAttempt(int maps, int reduces, boolean autoComplete,
@@ -500,6 +546,12 @@ public class TestJobHistoryParsing {
}
}
+ static class HistoryFileManagerForTest extends HistoryFileManager {
+ void deleteJobFromJobListCache(HistoryFileInfo fileInfo) {
+ jobListCache.delete(fileInfo);
+ }
+ }
+
public static void main(String[] args) throws Exception {
TestJobHistoryParsing t = new TestJobHistoryParsing();
t.testHistoryParsing();