You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tu...@apache.org on 2012/12/07 02:54:01 UTC
svn commit: r1418161 - in
/hadoop/common/branches/MR-2454/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/o...
Author: tucu
Date: Fri Dec 7 01:53:35 2012
New Revision: 1418161
URL: http://svn.apache.org/viewvc?rev=1418161&view=rev
Log:
Merge from trunk
Modified:
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/ (props changed)
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/CHANGES.txt (contents, props changed)
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/conf/ (props changed)
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java
hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (props changed)
Propchange: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1414747-1418159
Modified: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-mapreduce-project/CHANGES.txt?rev=1418161&r1=1418160&r2=1418161&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/MR-2454/hadoop-mapreduce-project/CHANGES.txt Fri Dec 7 01:53:35 2012
@@ -605,6 +605,15 @@ Release 0.23.6 - UNRELEASED
MAPREDUCE-4764. repair TestBinaryTokenFile (Ivan A. Veselovsky via bobby)
+ MAPREDUCE-4825. JobImpl.finished doesn't expect ERROR as a final job state
+ (jlowe via bobby)
+
+ MAPREDUCE-4817. Hardcoded task ping timeout kills tasks localizing large
+ amounts of data (tgraves)
+
+ MAPREDUCE-4836. Elapsed time for running tasks on AM web UI tasks page is 0
+ (Ravi Prakash via jeagles)
+
Release 0.23.5 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1414747-1418159
Propchange: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1414747-1418159
Modified: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1418161&r1=1418160&r2=1418161&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java Fri Dec 7 01:53:35 2012
@@ -274,7 +274,6 @@ public class TaskAttemptListenerImpl ext
@Override
public boolean ping(TaskAttemptID taskAttemptID) throws IOException {
LOG.info("Ping from " + taskAttemptID.toString());
- taskHeartbeatHandler.pinged(TypeConverter.toYarn(taskAttemptID));
return true;
}
Modified: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java?rev=1418161&r1=1418160&r2=1418161&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java Fri Dec 7 01:53:35 2012
@@ -46,33 +46,22 @@ import org.apache.hadoop.yarn.service.Ab
public class TaskHeartbeatHandler extends AbstractService {
private static class ReportTime {
- private long lastPing;
private long lastProgress;
public ReportTime(long time) {
setLastProgress(time);
}
- public synchronized void setLastPing(long time) {
- lastPing = time;
- }
-
public synchronized void setLastProgress(long time) {
lastProgress = time;
- lastPing = time;
- }
-
- public synchronized long getLastPing() {
- return lastPing;
}
-
+
public synchronized long getLastProgress() {
return lastProgress;
}
}
private static final Log LOG = LogFactory.getLog(TaskHeartbeatHandler.class);
- private static final int PING_TIMEOUT = 5 * 60 * 1000;
//thread which runs periodically to see the last time since a heartbeat is
//received from a task.
@@ -127,14 +116,6 @@ public class TaskHeartbeatHandler extend
}
}
- public void pinged(TaskAttemptId attemptID) {
- //only put for the registered attempts
- //TODO throw an exception if the task isn't registered.
- ReportTime time = runningAttempts.get(attemptID);
- if(time != null) {
- time.setLastPing(clock.getTime());
- }
- }
public void register(TaskAttemptId attemptID) {
runningAttempts.put(attemptID, new ReportTime(clock.getTime()));
@@ -159,10 +140,8 @@ public class TaskHeartbeatHandler extend
Map.Entry<TaskAttemptId, ReportTime> entry = iterator.next();
boolean taskTimedOut = (taskTimeOut > 0) &&
(currentTime > (entry.getValue().getLastProgress() + taskTimeOut));
- boolean pingTimedOut =
- (currentTime > (entry.getValue().getLastPing() + PING_TIMEOUT));
-
- if(taskTimedOut || pingTimedOut) {
+
+ if(taskTimedOut) {
// task is lost, remove from the list and raise lost event
iterator.remove();
eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(entry
Modified: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1418161&r1=1418160&r2=1418161&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Fri Dec 7 01:53:35 2012
@@ -828,6 +828,7 @@ public class JobImpl implements org.apac
case KILLED:
metrics.killedJob(this);
break;
+ case ERROR:
case FAILED:
metrics.failedJob(this);
break;
Modified: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java?rev=1418161&r1=1418160&r2=1418161&view=diff
==============================================================================
--- hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java (original)
+++ hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/TaskInfo.java Fri Dec 7 01:53:35 2012
@@ -59,11 +59,12 @@ public class TaskInfo {
TaskReport report = task.getReport();
this.startTime = report.getStartTime();
this.finishTime = report.getFinishTime();
- this.elapsedTime = Times.elapsed(this.startTime, this.finishTime, false);
+ this.state = report.getTaskState();
+ this.elapsedTime = Times.elapsed(this.startTime, this.finishTime,
+ this.state == TaskState.RUNNING);
if (this.elapsedTime == -1) {
this.elapsedTime = 0;
}
- this.state = report.getTaskState();
this.progress = report.getProgress() * 100;
this.id = MRApps.toString(task.getID());
this.taskNum = task.getID().getId();
Propchange: hadoop/common/branches/MR-2454/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1414747-1418159