You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ll...@apache.org on 2011/06/23 01:38:49 UTC
svn commit: r1138689 - in /hadoop/common/branches/MR-279/mapreduce: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/
mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/
Author: llu
Date: Wed Jun 22 23:38:49 2011
New Revision: 1138689
URL: http://svn.apache.org/viewvc?rev=1138689&view=rev
Log:
MAPREDUCE-2615. Make killJob go through AM and fix JobSummaryLog. (Siddharth Seth via llu)
Modified:
hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Wed Jun 22 23:38:49 2011
@@ -5,6 +5,9 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ MAPREDUCE-2615. Make killJob go through AM and fix JobSummaryLog.
+ (Siddharth Seth via llu)
+
Fix for NPE in YarnChild that was causing lots of tasks to fail. (vinodkv)
Fix for ConcurrentModification exception while iterating through tokens in
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java Wed Jun 22 23:38:49 2011
@@ -385,7 +385,7 @@ public class JobHistoryEventHandler exte
try {
HistoryEvent historyEvent = event.getHistoryEvent();
mi.writeEvent(historyEvent);
- processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary());
+ processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(), event.getJobID());
LOG.info("In HistoryEventHandler "
+ event.getHistoryEvent().getEventType());
} catch (IOException e) {
@@ -427,7 +427,7 @@ public class JobHistoryEventHandler exte
}
}
- private void processEventForJobSummary(HistoryEvent event, JobSummary summary) {
+ private void processEventForJobSummary(HistoryEvent event, JobSummary summary, JobId jobId) {
// context.getJob could be used for some of this info as well.
switch (event.getEventType()) {
case JOB_SUBMITTED:
@@ -467,9 +467,10 @@ public class JobHistoryEventHandler exte
case JOB_KILLED:
JobUnsuccessfulCompletionEvent juce = (JobUnsuccessfulCompletionEvent) event;
summary.setJobStatus(juce.getStatus());
+ summary.setNumFinishedMaps(context.getJob(jobId).getTotalMaps());
+ summary.setNumFinishedReduces(context.getJob(jobId).getTotalReduces());
+ summary.setJobFinishTime(juce.getFinishTime());
break;
- // TODO Verify: MRV2 + MRV1. A JOB_FINISHED event will always come in after
- // this. Stats on taskCounts can be set via that.
}
}
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ClientServiceDelegate.java Wed Jun 22 23:38:49 2011
@@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetJobReportRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskAttemptCompletionEventsRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.GetTaskReportsRequest;
+import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillJobRequest;
import org.apache.hadoop.mapreduce.v2.api.protocolrecords.KillTaskAttemptRequest;
import org.apache.hadoop.mapreduce.v2.api.records.JobReport;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
@@ -468,4 +469,38 @@ public class ClientServiceDelegate {
}
return true;
}
+
+ public boolean killJob(JobID oldJobID)
+ throws YarnRemoteException {
+ org.apache.hadoop.mapreduce.v2.api.records.JobId jobId
+ = TypeConverter.toYarn(oldJobID);
+ KillJobRequest killRequest = recordFactory.newRecordInstance(KillJobRequest.class);
+ MRClientProtocol protocol = getProxy(oldJobID);
+ if (protocol == null) {
+ return false;
+ }
+ try {
+ killRequest.setJobId(jobId);
+ protocol.killJob(killRequest);
+ return true;
+ } catch(YarnRemoteException yre) {//thrown by remote server, no need to redirect
+ LOG.warn(RPCUtil.toString(yre));
+ throw yre;
+ } catch(Exception e) {
+ // Not really requied - if this is always the history context.
+ LOG.debug("Failed to contact application master ", e);
+ MRClientProtocol proxy = getRefreshedProxy(oldJobID);
+ if (proxy == null) {
+ return false;
+ }
+ try {
+ killRequest.setJobId(jobId);
+ protocol.killJob(killRequest);
+ return true;
+ } catch(YarnRemoteException yre) {
+ LOG.warn(RPCUtil.toString(yre));
+ throw yre;
+ }
+ }
+ }
}
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1138689&r1=1138688&r2=1138689&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Wed Jun 22 23:38:49 2011
@@ -537,8 +537,10 @@ public class YARNRunner implements Clien
@Override
public void killJob(JobID arg0) throws IOException, InterruptedException {
+ if (!clientServiceDelegate.killJob(arg0)) {
resMgrDelegate.killApplication(TypeConverter.toYarn(arg0).getAppId());
}
+ }
@Override
public boolean killTask(TaskAttemptID arg0, boolean arg1) throws IOException,