You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2011/02/01 10:44:08 UTC
svn commit: r1065962 - in /hadoop/mapreduce/trunk: ./
src/java/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/mapreduce/security/token/
src/java/org/apache/hadoop/mapreduce/server/tasktracker/
Author: shv
Date: Tue Feb 1 09:44:08 2011
New Revision: 1065962
URL: http://svn.apache.org/viewvc?rev=1065962&view=rev
Log:
MAPREDUCE-2281. MR part of HADOOP-6642. Contributed by Chris Douglas, Po Cheung.
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=1065962&r1=1065961&r2=1065962&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Tue Feb 1 09:44:08 2011
@@ -516,6 +516,8 @@ Release 0.22.0 - Unreleased
preempt all tasks from one pool causing that pool to go below fairshare.
(Priyo Mustafi via shv)
+ MAPREDUCE-2281. MR part of HADOOP-6642. (Chris Douglas, Po Cheung via shv)
+
Release 0.21.1 - Unreleased
NEW FEATURES
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1065962&r1=1065961&r2=1065962&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Tue Feb 1 09:44:08 2011
@@ -23,8 +23,6 @@ import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.OutputStream;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
import java.nio.ByteBuffer;
import java.nio.IntBuffer;
import java.util.ArrayList;
@@ -34,20 +32,17 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.RawComparator;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.io.SequenceFile.CompressionType;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.serializer.Deserializer;
@@ -56,17 +51,14 @@ import org.apache.hadoop.io.serializer.S
import org.apache.hadoop.mapred.IFile.Writer;
import org.apache.hadoop.mapred.Merger.Segment;
import org.apache.hadoop.mapred.SortedRanges.SkipRangeIterator;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
-import org.apache.hadoop.mapreduce.split.JobSplit;
-import org.apache.hadoop.mapreduce.split.JobSplit.SplitMetaInfo;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
-import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitMetaInfo;
-import org.apache.hadoop.mapreduce.task.MapContextImpl;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.mapreduce.TaskCounter;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.map.WrappedMapper;
+import org.apache.hadoop.mapreduce.split.JobSplit.TaskSplitIndex;
+import org.apache.hadoop.mapreduce.task.MapContextImpl;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.IndexedSortable;
import org.apache.hadoop.util.IndexedSorter;
@@ -83,7 +75,6 @@ class MapTask extends Task {
public static final int MAP_OUTPUT_INDEX_RECORD_LENGTH = 24;
private TaskSplitIndex splitMetaInfo = new TaskSplitIndex();
- private String splitClass;
private final static int APPROX_HEADER_LENGTH = 150;
private static final Log LOG = LogFactory.getLog(MapTask.class.getName());
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?rev=1065962&r1=1065961&r2=1065962&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Tue Feb 1 09:44:08 2011
@@ -1087,7 +1087,7 @@ public class TaskTracker
// set the location of the token file into jobConf to transfer
// the name to TaskRunner
localJobConf.set(TokenCache.JOB_TOKENS_FILENAME,
- localJobTokenFile.toString());
+ localJobTokenFile);
// create the 'job-work' directory: job-specific shared directory for use as
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java?rev=1065962&r1=1065961&r2=1065962&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/TaskTrackerStatus.java Tue Feb 1 09:44:08 2011
@@ -427,8 +427,7 @@ public class TaskTrackerStatus implement
*/
public int countMapTasks() {
int mapCount = 0;
- for (Iterator<TaskStatus> it = taskReports.iterator(); it.hasNext();) {
- TaskStatus ts = it.next();
+ for (TaskStatus ts : taskReports) {
if (ts.getIsMap() && isTaskRunning(ts)) {
mapCount++;
}
@@ -442,8 +441,7 @@ public class TaskTrackerStatus implement
*/
public int countOccupiedMapSlots() {
int mapSlotsCount = 0;
- for (Iterator<TaskStatus> it = taskReports.iterator(); it.hasNext();) {
- TaskStatus ts = it.next();
+ for (TaskStatus ts : taskReports) {
if (ts.getIsMap() && isTaskRunning(ts)) {
mapSlotsCount += ts.getNumSlots();
}
@@ -465,8 +463,7 @@ public class TaskTrackerStatus implement
*/
public int countReduceTasks() {
int reduceCount = 0;
- for (Iterator<TaskStatus> it = taskReports.iterator(); it.hasNext();) {
- TaskStatus ts = it.next();
+ for (TaskStatus ts : taskReports) {
if ((!ts.getIsMap()) && isTaskRunning(ts)) {
reduceCount++;
}
@@ -480,8 +477,7 @@ public class TaskTrackerStatus implement
*/
public int countOccupiedReduceSlots() {
int reduceSlotsCount = 0;
- for (Iterator<TaskStatus> it = taskReports.iterator(); it.hasNext();) {
- TaskStatus ts = it.next();
+ for (TaskStatus ts : taskReports) {
if ((!ts.getIsMap()) && isTaskRunning(ts)) {
reduceSlotsCount += ts.getNumSlots();
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java?rev=1065962&r1=1065961&r2=1065962&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java Tue Feb 1 09:44:08 2011
@@ -81,6 +81,7 @@ public class DelegationTokenRenewal {
public void setTimerTask(TimerTask tTask) {
timerTask = tTask;
}
+ @Override
public String toString() {
return token + ";exp="+expirationDate;
}
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java?rev=1065962&r1=1065961&r2=1065962&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/tasktracker/Localizer.java Tue Feb 1 09:44:08 2011
@@ -253,7 +253,7 @@ public class Localizer {
if (!initStatus) {
throw new IOException("Not able to initialize attempt directories "
+ "in any of the configured local directories for the attempt "
- + attemptId.toString());
+ + attemptId);
}
}