You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by om...@apache.org on 2011/03/08 07:01:28 UTC
svn commit: r1079261 - in
/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop:
mapred/MapOutputFile.java mapred/MapTask.java mapred/Task.java
mapreduce/MRConfig.java
Author: omalley
Date: Tue Mar 8 06:01:27 2011
New Revision: 1079261
URL: http://svn.apache.org/viewvc?rev=1079261&view=rev
Log:
commit 94eae52b944fdc3dc3f1eba37d1dd189f1eb32c4
Author: Mahadev Konar <ma...@yahoo-inc.com>
Date: Fri Feb 4 12:56:49 2011 -0800
. MapOutputFile needs to be pluggable.
(chris douglas via mahadev)
+++ b/YAHOO-CHANGES.txt
+
+ . MapOutputFile needs to be pluggable.
+ (chris douglas via mahadev)
Modified:
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapOutputFile.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java
hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/MRConfig.java
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapOutputFile.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapOutputFile.java?rev=1079261&r1=1079260&r2=1079261&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapOutputFile.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapOutputFile.java Tue Mar 8 06:01:27 2011
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRConfig;
@@ -38,7 +39,7 @@ import org.apache.hadoop.mapreduce.MRCon
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class MapOutputFile {
+public class MapOutputFile implements Configurable {
private JobConf conf;
@@ -78,6 +79,13 @@ public class MapOutputFile {
}
/**
+ * Create a local map output file name on the same volume.
+ */
+ public Path getOutputFileForWriteInVolume(Path existing) {
+ return new Path(existing.getParent(), MAP_OUTPUT_FILENAME_STRING);
+ }
+
+ /**
* Return the path to a local map output index file created earlier
*
* @return path
@@ -104,6 +112,14 @@ public class MapOutputFile {
}
/**
+ * Create a local map output index file name on the same volume.
+ */
+ public Path getOutputIndexFileForWriteInVolume(Path existing) {
+ return new Path(existing.getParent(),
+ MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING);
+ }
+
+ /**
* Return a local map spill file created earlier.
*
* @param spillNumber the number
@@ -193,6 +209,7 @@ public class MapOutputFile {
conf.deleteLocalFiles(TaskTracker.OUTPUT);
}
+ @Override
public void setConf(Configuration conf) {
if (conf instanceof JobConf) {
this.conf = (JobConf) conf;
@@ -200,5 +217,10 @@ public class MapOutputFile {
this.conf = new JobConf(conf);
}
}
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java?rev=1079261&r1=1079260&r2=1079261&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/MapTask.java Tue Mar 8 06:01:27 2011
@@ -1682,13 +1682,13 @@ class MapTask extends Task {
}
if (numSpills == 1) { //the spill is the final output
rfs.rename(filename[0],
- new Path(filename[0].getParent(), "file.out"));
+ mapOutputFile.getOutputFileForWriteInVolume(filename[0]));
if (indexCacheList.size() == 0) {
rfs.rename(mapOutputFile.getSpillIndexFile(0),
- new Path(filename[0].getParent(),"file.out.index"));
+ mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]));
} else {
indexCacheList.get(0).writeToFile(
- new Path(filename[0].getParent(),"file.out.index"), job);
+ mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]), job);
}
return;
}
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java?rev=1079261&r1=1079260&r2=1079261&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapred/Task.java Tue Mar 8 06:01:27 2011
@@ -129,7 +129,7 @@ abstract public class Task implements Wr
private long initCpuCumulativeTime = 0;
protected JobConf conf;
- protected MapOutputFile mapOutputFile = new MapOutputFile();
+ protected MapOutputFile mapOutputFile;
protected LocalDirAllocator lDirAlloc;
private final static int MAX_RETRIES = 10;
protected JobContext jobContext;
@@ -1186,7 +1186,9 @@ abstract public class Task implements Wr
} else {
this.conf = new JobConf(conf);
}
- this.mapOutputFile.setConf(this.conf);
+ this.mapOutputFile = ReflectionUtils.newInstance(
+ conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
+ MapOutputFile.class, MapOutputFile.class), conf);
this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR);
// add the static resolutions (this is required for the junit to
// work on testcases that simulate multiple nodes on a single physical
Modified: hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/MRConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/MRConfig.java?rev=1079261&r1=1079260&r2=1079261&view=diff
==============================================================================
--- hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/MRConfig.java (original)
+++ hadoop/mapreduce/branches/yahoo-merge/src/java/org/apache/hadoop/mapreduce/MRConfig.java Tue Mar 8 06:01:27 2011
@@ -54,4 +54,6 @@ public interface MRConfig {
"mapreduce.cluster.delegation.token.max-lifetime";
public static final long DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT =
7*24*60*60*1000; // 7 days
+ public static final String TASK_LOCAL_OUTPUT_CLASS =
+ "mapreduce.task.local.output.class";
}