You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by rb...@apache.org on 2015/12/16 23:44:52 UTC
tez git commit: TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated
for DefaultSorter with multiple partitions (rbalamohan)
Repository: tez
Updated Branches:
refs/heads/branch-0.7 d2c3e5d21 -> 88cbd2edb
TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated for DefaultSorter with multiple partitions (rbalamohan)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/88cbd2ed
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/88cbd2ed
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/88cbd2ed
Branch: refs/heads/branch-0.7
Commit: 88cbd2edb6c8259f5dddd63eddc2505013f59bdf
Parents: d2c3e5d
Author: Rajesh Balamohan <rb...@apache.org>
Authored: Thu Dec 17 04:14:18 2015 +0530
Committer: Rajesh Balamohan <rb...@apache.org>
Committed: Thu Dec 17 04:14:18 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../library/common/sort/impl/dflt/DefaultSorter.java | 7 ++++++-
.../library/common/sort/impl/TestPipelinedSorter.java | 12 ++++++++++++
.../common/sort/impl/dflt/TestDefaultSorter.java | 4 ++--
4 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 71ab3ea..7eefa07 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES
TEZ-2949. Allow duplicate dag names within session for Tez.
ALL CHANGES
+ TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated for DefaultSorter with multiple partitions.
TEZ-3006. Remove unused import in TestHistoryParser.
TEZ-2918. Make progress notifications in IOs
TEZ-2979. FlakyTest: org.apache.tez.history.TestHistoryParser.
http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
index 6fe98b5..7cb7742 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
@@ -805,7 +805,6 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
} else {
if (numSpills > 0) {
additionalSpillBytesWritten.increment(compLength);
- numAdditionalSpills.increment(1);
// Reset the value will be set during the final merge.
outputBytesWithOverheadCounter.setValue(0);
} else {
@@ -910,6 +909,9 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
++numSpills;
if (!isFinalMergeEnabled()) {
numShuffleChunks.setValue(numSpills);
+ } else if (numSpills > 1) {
+ //Increment only when there was atleast one previous spill
+ numAdditionalSpills.increment(1);
}
} finally {
if (out != null) out.close();
@@ -982,6 +984,9 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
++numSpills;
if (!isFinalMergeEnabled()) {
numShuffleChunks.setValue(numSpills);
+ } else if (numSpills > 1) {
+ //Increment only when there is atleast one previous spill
+ numAdditionalSpills.increment(1);
}
} finally {
if (out != null) out.close();
http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
index f5076a6..1b92347 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
@@ -331,6 +331,18 @@ public class TestPipelinedSorter {
sorter.close();
}
+ @Test
+ public void testCountersWithMultiplePartitions() throws IOException {
+ conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
+ this.numOutputs = 5;
+ this.initialAvailableMem = 5 * 1024 * 1024;
+ PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
+ initialAvailableMem, 0);
+
+ writeData(sorter, 10000, 100);
+ verifyCounters(sorter, outputContext);
+ }
+
public void basicTest(int partitions, int numKeys, int keySize,
long initialAvailableMem, int blockSize) throws IOException {
this.numOutputs = partitions; // single output
http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
index 91112da..1da48c2 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
@@ -253,7 +253,7 @@ public class TestDefaultSorter {
conf.setLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 1);
context.requestInitialMemory(ExternalSorter.getInitialMemoryRequirement(conf,
context.getTotalMemoryAvailableToTask()), handler);
- DefaultSorter sorter = new DefaultSorter(context, conf, 1, handler.getMemoryAssigned());
+ DefaultSorter sorter = new DefaultSorter(context, conf, 5, handler.getMemoryAssigned());
//Write 1000 keys each of size 1000, (> 1 spill should happen)
try {
@@ -295,7 +295,7 @@ public class TestDefaultSorter {
MemoryUpdateCallbackHandler handler = new MemoryUpdateCallbackHandler();
context.requestInitialMemory(ExternalSorter.getInitialMemoryRequirement(conf,
context.getTotalMemoryAvailableToTask()), handler);
- DefaultSorter sorter = new DefaultSorter(context, conf, 1, handler.getMemoryAssigned());
+ DefaultSorter sorter = new DefaultSorter(context, conf, 5, handler.getMemoryAssigned());
//no data written. Empty
try {