You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by jl...@apache.org on 2013/05/13 22:54:59 UTC
svn commit: r1482100 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
Author: jlowe
Date: Mon May 13 20:54:58 2013
New Revision: 1482100
URL: http://svn.apache.org/r1482100
Log:
MAPREDUCE-5211. Reducer intermediate files can collide during merge. Contributed by Jason Lowe
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1482100&r1=1482099&r2=1482100&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon May 13 20:54:58 2013
@@ -24,6 +24,9 @@ Release 0.23.8 - UNRELEASED
MAPREDUCE-5147. Maven build should create
hadoop-mapreduce-client-app-VERSION.jar directly (Robert Parker via tgraves)
+ MAPREDUCE-5211. Reducer intermediate files can collide during merge
+ (jlowe)
+
Release 0.23.7 - 2013-04-18
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java?rev=1482100&r1=1482099&r2=1482100&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java Mon May 13 20:54:58 2013
@@ -505,7 +505,8 @@ public class MergeManager<K, V> {
}
private class OnDiskMerger extends MergeThread<MapOutput<K,V>,K,V> {
-
+ private int mergeCount = 0;
+
public OnDiskMerger(MergeManager<K, V> manager) {
super(manager, ioSortFactor, exceptionReporter);
setName("OnDiskMerger - Thread to merge on-disk map-outputs");
@@ -537,8 +538,10 @@ public class MergeManager<K, V> {
ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum);
// 2. Start the on-disk merge process
+ ++mergeCount;
Path outputPath =
- localDirAllocator.getLocalPathForWrite(inputs.get(0).toString(),
+ localDirAllocator.getLocalPathForWrite(
+ new Path(reduceId.toString(), "diskmerge" + mergeCount).toString(),
approxOutputSize, jobConf).suffix(Task.MERGED_OUTPUT_PREFIX);
Writer<K,V> writer =
new Writer<K,V>(jobConf, rfs, outputPath,