You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by yh...@apache.org on 2009/09/25 20:52:00 UTC
svn commit: r818952 - in /hadoop/mapreduce/branches/branch-0.21: ./ conf/
src/c++/ src/contrib/ src/contrib/capacity-scheduler/
src/contrib/data_join/ src/contrib/dynamic-scheduler/
src/contrib/eclipse-plugin/ src/contrib/fairscheduler/ src/contrib/ind...
Author: yhemanth
Date: Fri Sep 25 18:51:59 2009
New Revision: 818952
URL: http://svn.apache.org/viewvc?rev=818952&view=rev
Log:
MAPREDUCE-964. svn merge -r 818945:818946 from trunk to branch-0.21 to fix MAPREDUCE-964.
Added:
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
- copied unchanged from r818946, hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestTaskStatus.java
Modified:
hadoop/mapreduce/branches/branch-0.21/ (props changed)
hadoop/mapreduce/branches/branch-0.21/.gitignore (props changed)
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
hadoop/mapreduce/branches/branch-0.21/conf/ (props changed)
hadoop/mapreduce/branches/branch-0.21/conf/capacity-scheduler.xml.template (props changed)
hadoop/mapreduce/branches/branch-0.21/src/c++/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/build-contrib.xml (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/build.xml (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/data_join/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/dynamic-scheduler/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/eclipse-plugin/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/index/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/mrunit/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/CheckedEventQueue.java
hadoop/mapreduce/branches/branch-0.21/src/contrib/sqoop/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/contrib/vaidya/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/examples/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/java/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskStatus.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskTracker.java
hadoop/mapreduce/branches/branch-0.21/src/test/mapred/ (props changed)
hadoop/mapreduce/branches/branch-0.21/src/webapps/job/ (props changed)
Propchange: hadoop/mapreduce/branches/branch-0.21/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,2 +1,2 @@
/hadoop/core/branches/branch-0.19/mapred:713112
-/hadoop/mapreduce/trunk:818355,818918
+/hadoop/mapreduce/trunk:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/.gitignore
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/.gitignore:713112
/hadoop/core/trunk/.gitignore:784664-785643
-/hadoop/mapreduce/trunk/.gitignore:818355,818918
+/hadoop/mapreduce/trunk/.gitignore:818355,818918,818946
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=818952&r1=818951&r2=818952&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Fri Sep 25 18:51:59 2009
@@ -717,3 +717,7 @@
irrespective of slot size for the job.
(Ravi Gummadi via yhemanth)
+ MAPREDUCE-964. Fixed start and finish times of TaskStatus to be
+ consistent, thereby fixing inconsistencies in metering tasks.
+ (Sreekanth Ramakrishnan via yhemanth)
+
Propchange: hadoop/mapreduce/branches/branch-0.21/conf/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/conf:713112
/hadoop/core/trunk/conf:784664-785643
-/hadoop/mapreduce/trunk/conf:818355,818918
+/hadoop/mapreduce/trunk/conf:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/conf/capacity-scheduler.xml.template
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/conf/capacity-scheduler.xml.template:713112
/hadoop/core/trunk/conf/capacity-scheduler.xml.template:776175-785643
-/hadoop/mapreduce/trunk/conf/capacity-scheduler.xml.template:818355,818918
+/hadoop/mapreduce/trunk/conf/capacity-scheduler.xml.template:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/c++/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/c++:713112
/hadoop/core/trunk/src/c++:776175-784663
-/hadoop/mapreduce/trunk/src/c++:818355,818918
+/hadoop/mapreduce/trunk/src/c++:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib:713112
/hadoop/core/trunk/src/contrib:784664-785643
-/hadoop/mapreduce/trunk/src/contrib:818355,818918
+/hadoop/mapreduce/trunk/src/contrib:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/build-contrib.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build-contrib.xml:713112
/hadoop/core/trunk/src/contrib/build-contrib.xml:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/build-contrib.xml:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/build-contrib.xml:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/build.xml
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/build.xml:713112
/hadoop/core/trunk/src/contrib/build.xml:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/build.xml:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/build.xml:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/capacity-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/capacity-scheduler:713112
/hadoop/core/trunk/src/contrib/capacity-scheduler:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/capacity-scheduler:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/data_join/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/data_join:713112
/hadoop/core/trunk/src/contrib/data_join:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/data_join:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/data_join:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/dynamic-scheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/src/contrib/dynamic-scheduler:713112
/hadoop/core/trunk/src/contrib/dynamic-scheduler:784975-786373
-/hadoop/mapreduce/trunk/src/contrib/dynamic-scheduler:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/dynamic-scheduler:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/eclipse-plugin/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/core/src/contrib/eclipse-plugin:713112
/hadoop/core/trunk/src/contrib/eclipse-plugin:776175-784663
-/hadoop/mapreduce/trunk/src/contrib/eclipse-plugin:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/eclipse-plugin:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/fairscheduler/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/fairscheduler:713112
/hadoop/core/trunk/src/contrib/fairscheduler:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/fairscheduler:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/fairscheduler:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/index/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/index:713112
/hadoop/core/trunk/src/contrib/index:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/index:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/index:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/mrunit/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/mrunit:713112
/hadoop/core/trunk/src/contrib/mrunit:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/mrunit:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/mrunit:818355,818918,818946
Modified: hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/CheckedEventQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/CheckedEventQueue.java?rev=818952&r1=818951&r2=818952&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/CheckedEventQueue.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/contrib/mumak/src/test/org/apache/hadoop/mapred/CheckedEventQueue.java Fri Sep 25 18:51:59 2009
@@ -161,6 +161,7 @@
org.apache.hadoop.mapred.TaskAttemptID.downgrade(taskId);
MapTaskStatus status = new MapTaskStatus(taskIdOldApi, 1.0f, 1,
State.SUCCEEDED, null, null, null, Phase.MAP, null);
+ status.setStartTime(mapStart);
status.setFinishTime(mapDone);
TaskAttemptCompletionEvent completionEvent =
new TaskAttemptCompletionEvent(taskTracker, status);
@@ -177,6 +178,7 @@
org.apache.hadoop.mapred.TaskAttemptID.downgrade(taskId);
ReduceTaskStatus status = new ReduceTaskStatus(taskIdOldApi, 1.0f, 1,
State.SUCCEEDED, null, null, null, Phase.REDUCE, null);
+ status.setStartTime(mapDone);
status.setFinishTime(reduceDone);
TaskAttemptCompletionEvent completionEvent =
new TaskAttemptCompletionEvent(taskTracker, status);
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/sqoop/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/src/contrib/sqoop:713112
/hadoop/core/trunk/src/contrib/sqoop:784975-786373
-/hadoop/mapreduce/trunk/src/contrib/sqoop:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/sqoop:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/streaming/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/streaming:713112
/hadoop/core/trunk/src/contrib/streaming:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/streaming:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/streaming:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/contrib/vaidya/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/contrib/vaidya:713112
/hadoop/core/trunk/src/contrib/vaidya:776175-786373
-/hadoop/mapreduce/trunk/src/contrib/vaidya:818355,818918
+/hadoop/mapreduce/trunk/src/contrib/vaidya:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/examples/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/examples:713112
/hadoop/core/trunk/src/examples:776175-784663
-/hadoop/mapreduce/trunk/src/examples:818355,818918
+/hadoop/mapreduce/trunk/src/examples:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/java:713112
/hadoop/core/trunk/src/mapred:776175-785643
-/hadoop/mapreduce/trunk/src/java:818355,818918
+/hadoop/mapreduce/trunk/src/java:818355,818918,818946
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskStatus.java?rev=818952&r1=818951&r2=818952&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskStatus.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskStatus.java Fri Sep 25 18:51:59 2009
@@ -27,6 +27,7 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringUtils;
/**************************************************
* Describes the current status of a task. This is
* not intended to be a comprehensive piece of data.
@@ -132,11 +133,21 @@
}
/**
- * Sets finishTime.
+ * Sets finishTime for the task status if and only if the
+ * start time is set and passed finish time is greater than
+ * zero.
+ *
* @param finishTime finish time of task.
*/
void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
+ if(this.getStartTime() > 0 && finishTime > 0) {
+ this.finishTime = finishTime;
+ } else {
+ //Using String utils to get the stack trace.
+ LOG.error("Trying to set finish time for task " + taskid +
+ " when no start time is set, stackTrace is : " +
+ StringUtils.stringifyException(new Exception()));
+ }
}
/**
* Get shuffle finish time for the task. If shuffle finish time was
@@ -201,11 +212,20 @@
}
/**
- * Set startTime of the task.
+ * Set startTime of the task if start time is greater than zero.
* @param startTime start time
*/
void setStartTime(long startTime) {
- this.startTime = startTime;
+ //Making the assumption of passed startTime to be a positive
+ //long value explicit.
+ if (startTime > 0) {
+ this.startTime = startTime;
+ } else {
+ //Using String utils to get the stack trace.
+ LOG.error("Trying to set illegal startTime for task : " + taskid +
+ ".Stack trace is : " +
+ StringUtils.stringifyException(new Exception()));
+ }
}
/**
* Get current phase of this task. Phase.Map in case of map tasks,
@@ -326,11 +346,11 @@
setDiagnosticInfo(status.getDiagnosticInfo());
- if (status.getStartTime() != 0) {
- this.startTime = status.getStartTime();
+ if (status.getStartTime() > 0) {
+ this.setStartTime(status.getStartTime());
}
- if (status.getFinishTime() != 0) {
- this.finishTime = status.getFinishTime();
+ if (status.getFinishTime() > 0) {
+ this.setFinishTime(status.getFinishTime());
}
this.phase = status.getPhase();
@@ -359,8 +379,8 @@
setProgress(progress);
setStateString(state);
setPhase(phase);
- if (finishTime != 0) {
- this.finishTime = finishTime;
+ if (finishTime > 0) {
+ setFinishTime(finishTime);
}
}
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=818952&r1=818951&r2=818952&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskTracker.java Fri Sep 25 18:51:59 2009
@@ -2487,6 +2487,7 @@
taskStatus.setRunState(TaskStatus.State.KILLED);
}
}
+ taskStatus.setFinishTime(System.currentTimeMillis());
removeFromMemoryManager(task.getTaskID());
releaseSlot();
}
Propchange: hadoop/mapreduce/branches/branch-0.21/src/test/mapred/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/test/mapred:713112
/hadoop/core/trunk/src/test/mapred:776175-785643
-/hadoop/mapreduce/trunk/src/test/mapred:818355,818918
+/hadoop/mapreduce/trunk/src/test/mapred:818355,818918,818946
Propchange: hadoop/mapreduce/branches/branch-0.21/src/webapps/job/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 25 18:51:59 2009
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/src/webapps/job:713112
/hadoop/core/trunk/src/webapps/job:776175-785643
-/hadoop/mapreduce/trunk/src/webapps/job:818355,818918
+/hadoop/mapreduce/trunk/src/webapps/job:818355,818918,818946