You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ha...@apache.org on 2020/08/24 21:12:40 UTC
[tez] branch master updated: TEZ-4216 : RLE check in
MergeManager::finalMerge could be disabled (Rajesh Balamohan via Ashutosh
Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 3986f5b TEZ-4216 : RLE check in MergeManager::finalMerge could be disabled (Rajesh Balamohan via Ashutosh Chauhan)
3986f5b is described below
commit 3986f5bc2c2e34ad160f77afd7478b26935fffa5
Author: Rajesh Balamohan <rb...@apache.org>
AuthorDate: Mon Aug 24 14:11:52 2020 -0700
TEZ-4216 : RLE check in MergeManager::finalMerge could be disabled (Rajesh Balamohan via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
.../common/shuffle/orderedgrouped/MergeManager.java | 8 ++++----
.../tez/runtime/library/common/sort/impl/TezMerger.java | 15 +++++++++++++++
2 files changed, 19 insertions(+), 4 deletions(-)
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index 70f9e55..dd23b3b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -1316,10 +1316,10 @@ public class MergeManager implements FetchedInputAllocatorOrderedGrouped {
LOG.info(finalMergeLog.toString());
}
// This is doing nothing but creating an iterator over the segments.
- return TezMerger.merge(job, fs, keyClass, valueClass,
- finalSegments, finalSegments.size(), tmpDir,
- comparator, progressable, spilledRecordsCounter, null,
- additionalBytesRead, null);
+ return TezMerger.merge(job, fs, keyClass, valueClass, codec,
+ finalSegments, finalSegments.size(), tmpDir,
+ comparator, progressable, spilledRecordsCounter, null,
+ additionalBytesRead, null);
}
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
index 3e04e74..726810b 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
@@ -128,6 +128,21 @@ public class TezMerger {
bytesReadCounter, mergePhase);
}
+ public static TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
+ Class keyClass, Class valueClass, CompressionCodec codec,
+ List<Segment> segments,
+ int mergeFactor, Path tmpDir,
+ RawComparator comparator, Progressable reporter,
+ TezCounter readsCounter,
+ TezCounter writesCounter,
+ TezCounter bytesReadCounter,
+ Progress mergePhase) throws IOException, InterruptedException {
+ return new MergeQueue(conf, fs, segments, comparator, reporter,
+ false, codec, false, false)
+ .merge(keyClass, valueClass, mergeFactor, tmpDir,
+ readsCounter, writesCounter, bytesReadCounter, mergePhase);
+ }
+
public static <K extends Object, V extends Object>
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
Class keyClass, Class valueClass,