You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/03/27 18:41:59 UTC
git commit: TEZ-981. Port MAPREDUCE-3685. (sseth)
Repository: incubator-tez
Updated Branches:
refs/heads/master 57b61c22f -> cde19354d
TEZ-981. Port MAPREDUCE-3685. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/cde19354
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/cde19354
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/cde19354
Branch: refs/heads/master
Commit: cde19354d5a7ee73a2fff5312d5d5ecb995da428
Parents: 57b61c2
Author: Siddharth Seth <ss...@apache.org>
Authored: Thu Mar 27 10:41:40 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Thu Mar 27 10:41:40 2014 -0700
----------------------------------------------------------------------
.../common/shuffle/impl/MergeManager.java | 4 ++--
.../library/common/sort/impl/TezMerger.java | 22 --------------------
2 files changed, 2 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cde19354/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
index 5a952ba..8a09aab 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
@@ -634,7 +634,7 @@ public class MergeManager {
private class OnDiskMerger extends MergeThread<Path> {
public OnDiskMerger(MergeManager manager) {
- super(manager, Integer.MAX_VALUE, exceptionReporter);
+ super(manager, ioSortFactor, exceptionReporter);
setName("DiskToDiskMerger [" + TezUtils.cleanVertexName(inputContext.getSourceVertexName()) + "]");
setDaemon(true);
}
@@ -888,7 +888,7 @@ public class MergeManager {
diskSegments.addAll(0, memDiskSegments);
memDiskSegments.clear();
TezRawKeyValueIterator diskMerge = TezMerger.merge(
- job, fs, keyClass, valueClass, diskSegments,
+ job, fs, keyClass, valueClass, codec, diskSegments,
ioSortFactor, numInMemSegments, tmpDir, comparator,
nullProgressable, false, spilledRecordsCounter, null, additionalBytesRead, null);
diskSegments.clear();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cde19354/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
index ae03370..ee62b70 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
@@ -171,28 +171,6 @@ public class TezMerger {
}
public static <K extends Object, V extends Object>
- TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
- Class keyClass, Class valueClass,
- List<Segment> segments,
- int mergeFactor, int inMemSegments, Path tmpDir,
- RawComparator comparator, Progressable reporter,
- boolean sortSegments,
- TezCounter readsCounter,
- TezCounter writesCounter,
- TezCounter bytesReadCounter,
- Progress mergePhase)
- throws IOException {
- return new MergeQueue(conf, fs, segments, comparator, reporter,
- sortSegments, false).merge(keyClass, valueClass,
- mergeFactor, inMemSegments,
- tmpDir,
- readsCounter, writesCounter,
- bytesReadCounter,
- mergePhase);
- }
-
-
- static <K extends Object, V extends Object>
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
Class keyClass, Class valueClass,
CompressionCodec codec,