You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2010/06/08 00:14:21 UTC
svn commit: r952462 - in /hadoop/mapreduce/branches/branch-0.21: ./
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapred/lib/
Author: tomwhite
Date: Mon Jun 7 22:14:20 2010
New Revision: 952462
URL: http://svn.apache.org/viewvc?rev=952462&view=rev
Log:
Merge -r 952459:952460 from trunk to branch-0.21. Fixes: MAPREDUCE-1725
Modified:
hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (contents, props changed)
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java
Modified: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/CHANGES.txt?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/branch-0.21/CHANGES.txt Mon Jun 7 22:14:20 2010
@@ -1584,3 +1584,6 @@ Release 0.21.0 - Unreleased
MAPREDUCE-913. TaskRunner crashes with NPE resulting in held up slots,
UNINITIALIZED tasks and hung TaskTracker. (Amareshwari Sriramadasu and
Sreekanth Ramakrishnan via vinodkv)
+
+ MAPREDUCE-1725. Fix MapReduce API incompatibilities between 0.20 and 0.21.
+ (tomwhite)
Propchange: hadoop/mapreduce/branches/branch-0.21/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon Jun 7 22:14:20 2010
@@ -1,3 +1,3 @@
/hadoop/core/branches/branch-0.19/mapred/CHANGES.txt:713112
/hadoop/mapreduce/branches/HDFS-641/CHANGES.txt:817878-835964
-/hadoop/mapreduce/trunk/CHANGES.txt:940364,942764,946526
+/hadoop/mapreduce/trunk/CHANGES.txt:940364,942764,946526,952460
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/FileOutputFormat.java Mon Jun 7 22:14:20 2010
@@ -26,9 +26,9 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
/** A base class for {@link OutputFormat}. */
@InterfaceAudience.Public
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskAttemptID.java Mon Jun 7 22:14:20 2010
@@ -63,6 +63,22 @@ public class TaskAttemptID extends org.a
* Constructs a TaskId object from given parts.
* @param jtIdentifier jobTracker identifier
* @param jobId job number
+ * @param isMap whether the tip is a map
+ * @param taskId taskId number
+ * @param id the task attempt number
+ * @deprecated Use {@link #TaskAttemptID(String, int, TaskType, int, int)}.
+ */
+ @Deprecated
+ public TaskAttemptID(String jtIdentifier, int jobId, boolean isMap,
+ int taskId, int id) {
+ this(jtIdentifier, jobId, isMap ? TaskType.MAP : TaskType.REDUCE, taskId,
+ id);
+ }
+
+ /**
+ * Constructs a TaskId object from given parts.
+ * @param jtIdentifier jobTracker identifier
+ * @param jobId job number
* @param type the TaskType
* @param taskId taskId number
* @param id the task attempt number
@@ -122,6 +138,31 @@ public class TaskAttemptID extends org.a
* of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first
* map task</i>, we would use :
* <pre>
+ * TaskAttemptID.getTaskAttemptIDsPattern(null, null, true, 1, null);
+ * </pre>
+ * which will return :
+ * <pre> "attempt_[^_]*_[0-9]*_m_000001_[0-9]*" </pre>
+ * @param jtIdentifier jobTracker identifier, or null
+ * @param jobId job number, or null
+ * @param isMap whether the tip is a map, or null
+ * @param taskId taskId number, or null
+ * @param attemptId the task attempt number, or null
+ * @return a regex pattern matching TaskAttemptIDs
+ */
+ @Deprecated
+ public static String getTaskAttemptIDsPattern(String jtIdentifier,
+ Integer jobId, Boolean isMap, Integer taskId, Integer attemptId) {
+ return getTaskAttemptIDsPattern(jtIdentifier, jobId,
+ isMap ? TaskType.MAP : TaskType.REDUCE, taskId, attemptId);
+ }
+
+ /**
+ * Returns a regex pattern which matches task attempt IDs. Arguments can
+ * be given null, in which case that part of the regex will be generic.
+ * For example to obtain a regex matching <i>all task attempt IDs</i>
+ * of <i>any jobtracker</i>, in <i>any job</i>, of the <i>first
+ * map task</i>, we would use :
+ * <pre>
* TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null);
* </pre>
* which will return :
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/TaskID.java Mon Jun 7 22:14:20 2010
@@ -55,6 +55,32 @@ public class TaskID extends org.apache.h
/**
* Constructs a TaskID object from given {@link JobID}.
* @param jobId JobID that this tip belongs to
+ * @param isMap whether the tip is a map
+ * @param id the tip number
+ * @deprecated Use {@link #TaskID(String, int, TaskType, int)}
+ */
+ @Deprecated
+ public TaskID(org.apache.hadoop.mapreduce.JobID jobId, boolean isMap,int id) {
+ this(jobId, isMap ? TaskType.MAP : TaskType.REDUCE, id);
+ }
+
+ /**
+ * Constructs a TaskInProgressId object from given parts.
+ * @param jtIdentifier jobTracker identifier
+ * @param jobId job number
+ * @param isMap whether the tip is a map
+ * @param id the tip number
+ * @deprecated Use {@link #TaskID(org.apache.hadoop.mapreduce.JobID, TaskType,
+ * int)}
+ */
+ @Deprecated
+ public TaskID(String jtIdentifier, int jobId, boolean isMap, int id) {
+ this(jtIdentifier, jobId, isMap ? TaskType.MAP : TaskType.REDUCE, id);
+ }
+
+ /**
+ * Constructs a TaskID object from given {@link JobID}.
+ * @param jobId JobID that this tip belongs to
* @param type the {@link TaskType}
* @param id the tip number
*/
@@ -114,6 +140,31 @@ public class TaskID extends org.apache.h
* <pre> "task_[^_]*_[0-9]*_m_000001*" </pre>
* @param jtIdentifier jobTracker identifier, or null
* @param jobId job number, or null
+ * @param isMap whether the tip is a map, or null
+ * @param taskId taskId number, or null
+ * @return a regex pattern matching TaskIDs
+ * @deprecated Use {@link TaskID#getTaskIDsPattern(String, Integer, TaskType,
+ * Integer)}
+ */
+ @Deprecated
+ public static String getTaskIDsPattern(String jtIdentifier, Integer jobId
+ , Boolean isMap, Integer taskId) {
+ return getTaskIDsPattern(jtIdentifier, jobId,
+ isMap ? TaskType.MAP : TaskType.REDUCE, taskId);
+ }
+
+ /**
+ * Returns a regex pattern which matches task IDs. Arguments can
+ * be given null, in which case that part of the regex will be generic.
+ * For example to obtain a regex matching <i>the first map task</i>
+ * of <i>any jobtracker</i>, of <i>any job</i>, we would use :
+ * <pre>
+ * TaskID.getTaskIDsPattern(null, null, true, 1);
+ * </pre>
+ * which will return :
+ * <pre> "task_[^_]*_[0-9]*_m_000001*" </pre>
+ * @param jtIdentifier jobTracker identifier, or null
+ * @param jobId job number, or null
* @param type the {@link TaskType}, or null
* @param taskId taskId number, or null
* @return a regex pattern matching TaskIDs
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileInputFormat.java Mon Jun 7 22:14:20 2010
@@ -19,9 +19,11 @@
package org.apache.hadoop.mapred.lib;
import java.io.IOException;
+import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@ -68,6 +70,27 @@ public abstract class CombineFileInputFo
throws IOException {
return super.getSplits(new Job(job)).toArray(new InputSplit[0]);
}
+
+ /**
+ * Create a new pool and add the filters to it.
+ * A split cannot have files from different pools.
+ * @deprecated Use {@link #createPool(List)}.
+ */
+ @Deprecated
+ protected void createPool(JobConf conf, List<PathFilter> filters) {
+ createPool(filters);
+ }
+
+ /**
+ * Create a new pool and add the filters to it.
+ * A pathname can satisfy any one of the specified filters.
+ * A split cannot have files from different pools.
+ * @deprecated Use {@link #createPool(PathFilter...)}.
+ */
+ @Deprecated
+ protected void createPool(JobConf conf, PathFilter... filters) {
+ createPool(filters);
+ }
/**
* This is not implemented yet.
Modified: hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java?rev=952462&r1=952461&r2=952462&view=diff
==============================================================================
--- hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java (original)
+++ hadoop/mapreduce/branches/branch-0.21/src/java/org/apache/hadoop/mapred/lib/CombineFileSplit.java Mon Jun 7 22:14:20 2010
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapred.lib;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
@@ -39,11 +41,24 @@ public class CombineFileSplit extends
public CombineFileSplit() {
}
+
+ public CombineFileSplit(JobConf job, Path[] files, long[] start,
+ long[] lengths, String[] locations) {
+ super(files, start, lengths, locations);
+ this.job = job;
+ }
public CombineFileSplit(JobConf job, Path[] files, long[] lengths) {
super(files, lengths);
this.job = job;
}
+
+ /**
+ * Copy constructor
+ */
+ public CombineFileSplit(CombineFileSplit old) throws IOException {
+ super(old);
+ }
public JobConf getJob() {
return job;