You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2011/04/20 11:48:11 UTC
svn commit: r1095343 - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/
mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/
mr-clien...
Author: sharad
Date: Wed Apr 20 09:48:10 2011
New Revision: 1095343
URL: http://svn.apache.org/viewvc?rev=1095343&view=rev
Log:
Fix race condition in TestJobHistoryEvents and TestJobHistoryParsing.
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Wed Apr 20 09:48:10 2011
@@ -3,6 +3,7 @@ Hadoop MapReduce Change Log
Trunk (unreleased changes)
MAPREDUCE-279
+ Fix race condition in TestJobHistoryEvents and TestJobHistoryParsing. (sharad)
Added functionality to stop/start queues. (acmurthy)
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Wed Apr 20 09:48:10 2011
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -36,9 +37,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.v2.YarnMRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
-import org.apache.hadoop.mapreduce.v2.YarnMRJobConfig;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.conf.YARNApplicationConstants;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -147,12 +148,33 @@ public class JobHistoryEventHandler exte
@Override
public void stop() {
stopped = true;
- eventHandlingThread.interrupt();
+ //do not interrupt while event handling is in progress
+ synchronized(this) {
+ eventHandlingThread.interrupt();
+ }
+
try {
eventHandlingThread.join();
} catch (InterruptedException ie) {
LOG.info("Interruped Exception while stopping", ie);
}
+ //write all the events remaining in queue
+ Iterator<JobHistoryEvent> it = eventQueue.iterator();
+ while(it.hasNext()) {
+ JobHistoryEvent ev = it.next();
+ LOG.info("In stop, writing event " + ev.getType());
+ handleEvent(ev);
+ }
+
+ //close all file handles
+ for (MetaInfo mi : fileMap.values()) {
+ try {
+ mi.closeWriter();
+ } catch (IOException e) {
+ LOG.info("Exception while closing file " + e.getMessage());
+ }
+ }
+
super.stop();
}
@@ -219,7 +241,7 @@ public class JobHistoryEventHandler exte
}
}
- protected void handleEvent(JobHistoryEvent event) {
+ protected synchronized void handleEvent(JobHistoryEvent event) {
// check for first event from a job
if (event.getHistoryEvent().getEventType() == EventType.JOB_SUBMITTED) {
try {
@@ -314,6 +336,7 @@ public class JobHistoryEventHandler exte
synchronized void writeEvent(HistoryEvent event) throws IOException {
if (writer != null) {
writer.write(event);
+ writer.flush();
}
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobHistoryParsing.java Wed Apr 20 09:48:10 2011
@@ -53,6 +53,7 @@ public class TestJobHistoryParsing {
JobId jobId = job.getID();
LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
app.waitForState(job, JobState.SUCCEEDED);
+ app.stop();
String jobhistoryFileName = TypeConverter.fromYarn(jobId).toString();
String user =
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java?rev=1095343&r1=1095342&r2=1095343&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestJobHistoryEvents.java Wed Apr 20 09:48:10 2011
@@ -58,6 +58,7 @@ public class TestJobHistoryEvents {
JobId jobId = job.getID();
LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
app.waitForState(job, JobState.SUCCEEDED);
+ app.stop();
/*
* Use HistoryContext to read logged events and verify the number of
* completed maps