You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 04:29:46 UTC
svn commit: r1077006 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
mapred/org/apache/hadoop/mapred/TaskStatus.java
test/org/apache/hadoop/mapred/TestTaskStatus.java
Author: omalley
Date: Fri Mar 4 03:29:46 2011
New Revision: 1077006
URL: http://svn.apache.org/viewvc?rev=1077006&view=rev
Log:
commit d31ea1a98e2c16c9a9afc9beb9d63568fe32f81e
Author: Arun C Murthy <ac...@apache.org>
Date: Mon Sep 28 13:36:31 2009 -0700
MAPREDUCE-964. Fixed start and finish times of TaskStatus to be consistent, thereby fixing inconsistencies in metering tasks. Contributed by Sreekanth Ramakrishnan.
from: http://issues.apache.org/jira/secure/attachment/12420539/mapreduce-964-ydist.patch
+++ b/YAHOO-CHANGES.txt
+57. MAPREDUCE-964. Fixed start and finish times of TaskStatus to be
+ consistent, thereby fixing inconsistencies in metering tasks.
+ Contributed by Sreekanth Ramakrishnan.
+
Added:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestTaskStatus.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskStatus.java?rev=1077006&r1=1077005&r2=1077006&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskStatus.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/TaskStatus.java Fri Mar 4 03:29:46 2011
@@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.util.StringUtils;
/**************************************************
* Describes the current status of a task. This is
* not intended to be a comprehensive piece of data.
@@ -130,12 +131,23 @@ abstract class TaskStatus implements Wri
}
/**
- * Sets finishTime.
+ * Sets finishTime for the task status if and only if the
+ * start time is set and passed finish time is greater than
+ * zero.
+ *
* @param finishTime finish time of task.
*/
void setFinishTime(long finishTime) {
- this.finishTime = finishTime;
+ if(this.getStartTime() > 0 && finishTime > 0) {
+ this.finishTime = finishTime;
+ } else {
+ //Using String utils to get the stack trace.
+ LOG.error("Trying to set finish time for task " + taskid +
+ " when no start time is set, stackTrace is : " +
+ StringUtils.stringifyException(new Exception()));
+ }
}
+
/**
* Get shuffle finish time for the task. If shuffle finish time was
* not set due to shuffle/sort/finish phases ending within same
@@ -181,12 +193,22 @@ abstract class TaskStatus implements Wri
}
/**
- * Set startTime of the task.
+ * Set startTime of the task if start time is greater than zero.
* @param startTime start time
*/
void setStartTime(long startTime) {
- this.startTime = startTime;
+ //Making the assumption of passed startTime to be a positive
+ //long value explicit.
+ if (startTime > 0) {
+ this.startTime = startTime;
+ } else {
+ //Using String utils to get the stack trace.
+ LOG.error("Trying to set illegal startTime for task : " + taskid +
+ ".Stack trace is : " +
+ StringUtils.stringifyException(new Exception()));
+ }
}
+
/**
* Get current phase of this task. Phase.Map in case of map tasks,
* for reduce one of Phase.SHUFFLE, Phase.SORT or Phase.REDUCE.
@@ -301,11 +323,11 @@ abstract class TaskStatus implements Wri
setDiagnosticInfo(status.getDiagnosticInfo());
- if (status.getStartTime() != 0) {
+ if (status.getStartTime() > 0) {
this.startTime = status.getStartTime();
}
- if (status.getFinishTime() != 0) {
- this.finishTime = status.getFinishTime();
+ if (status.getFinishTime() > 0) {
+ setFinishTime(status.getFinishTime());
}
this.phase = status.getPhase();
@@ -334,8 +356,8 @@ abstract class TaskStatus implements Wri
setProgress(progress);
setStateString(state);
setPhase(phase);
- if (finishTime != 0) {
- this.finishTime = finishTime;
+ if (finishTime > 0) {
+ setFinishTime(finishTime);
}
}
Added: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestTaskStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestTaskStatus.java?rev=1077006&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestTaskStatus.java (added)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestTaskStatus.java Fri Mar 4 03:29:46 2011
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.mapred;
+
+import junit.framework.TestCase;
+
+public class TestTaskStatus extends TestCase {
+
+ public void testMapTaskStatusStartAndFinishTimes() {
+ checkTaskStatues(true);
+ }
+
+ public void testReduceTaskStatusStartAndFinishTimes() {
+ checkTaskStatues(false);
+ }
+
+ /**
+ * Private utility method which ensures uniform testing of newly created
+ * TaskStatus object.
+ *
+ * @param isMap
+ * true to test map task status, false for reduce.
+ */
+ private void checkTaskStatues(boolean isMap) {
+
+ TaskStatus status = null;
+ if (isMap) {
+ status = new MapTaskStatus();
+ } else {
+ status = new ReduceTaskStatus();
+ }
+ long currentTime = System.currentTimeMillis();
+ // first try to set the finish time before
+ // start time is set.
+ status.setFinishTime(currentTime);
+ assertEquals("Finish time of the task status set without start time", 0,
+ status.getFinishTime());
+ // Now set the start time to right time.
+ status.setStartTime(currentTime);
+ assertEquals("Start time of the task status not set correctly.",
+ currentTime, status.getStartTime());
+ // try setting wrong start time to task status.
+ long wrongTime = -1;
+ status.setStartTime(wrongTime);
+ assertEquals(
+ "Start time of the task status is set to wrong negative value",
+ currentTime, status.getStartTime());
+ // finally try setting wrong finish time i.e. negative value.
+ status.setFinishTime(wrongTime);
+ assertEquals("Finish time of task status is set to wrong negative value",
+ 0, status.getFinishTime());
+ status.setFinishTime(currentTime);
+ assertEquals("Finish time of the task status not set correctly.",
+ currentTime, status.getFinishTime());
+ }
+}