You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/12/16 21:53:41 UTC
tez git commit: TEZ-1836. Provide better error messages when
tez.runtime.io.sort.mb,
spill percentage is incorrectly configured. Contributed by Vasanth kumar RJ.
Repository: tez
Updated Branches:
refs/heads/master 0f462d23a -> 665801c6d
TEZ-1836. Provide better error messages when tez.runtime.io.sort.mb,
spill percentage is incorrectly configured. Contributed by Vasanth kumar
RJ.
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/665801c6
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/665801c6
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/665801c6
Branch: refs/heads/master
Commit: 665801c6d4d3721d2b6d82df3a2e009548e209c3
Parents: 0f462d2
Author: Siddharth Seth <ss...@apache.org>
Authored: Tue Dec 16 12:53:16 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Tue Dec 16 12:53:16 2014 -0800
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../common/sort/impl/ExternalSorter.java | 8 ++-
.../common/sort/impl/PipelinedSorter.java | 14 +-----
.../common/sort/impl/dflt/DefaultSorter.java | 13 ++---
.../library/output/TestOnFileSortedOutput.java | 53 ++++++++++++++++++++
5 files changed, 68 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/665801c6/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b81cfd9..15551db 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -64,6 +64,9 @@ TEZ-UI CHANGES (TEZ-8):
TEZ-1783. Wrapper in standalone mode.
TEZ-1820. Fix wrong links.
+Release 0.5.4: Unreleased
+ TEZ-1836. Provide better error messages when tez.runtime.io.sort.mb, spill percentage is incorrectly configured.
+
Release 0.5.3: Unreleased
ALL CHANGES:
http://git-wip-us.apache.org/repos/asf/tez/blob/665801c6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
index c73fc49..9aee53a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/ExternalSorter.java
@@ -286,9 +286,13 @@ public abstract class ExternalSorter {
conf.getInt(
TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB,
TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB_DEFAULT);
- Preconditions.checkArgument(initialMemRequestMb != 0, "io.sort.mb should be larger than 0");
+ Preconditions.checkArgument(initialMemRequestMb > 0 && initialMemRequestMb <= 2047,
+ TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB
+ + " should be larger than 0 and less than or equal to 2047");
long reqBytes = ((long) initialMemRequestMb) << 20;
- LOG.info("Requested SortBufferSize (io.sort.mb): " + initialMemRequestMb);
+ LOG.info("Requested SortBufferSize ("
+ + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + "): "
+ + initialMemRequestMb);
return reqBytes;
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/665801c6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index afa2370..e353c00 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -100,22 +100,10 @@ public class PipelinedSorter extends ExternalSorter {
partitionBits = bitcount(partitions)+1;
//sanity checks
- final float spillper =
- this.conf.getFloat(
- TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT,
- TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT);
final int sortmb = this.availableMemoryMb;
indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT);
- if (spillper > (float)1.0 || spillper <= (float)0.0) {
- throw new IOException("Invalid \"" + TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT +
- "\": " + spillper);
- }
- if ((sortmb & 0x7FF) != sortmb) {
- throw new IOException(
- "Invalid \"" + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB + "\": " + sortmb);
- }
-
+
// buffers and accounting
int maxMemUsage = sortmb << 20;
maxMemUsage -= maxMemUsage % METASIZE;
http://git-wip-us.apache.org/repos/asf/tez/blob/665801c6/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
index 82ae225..f872e1f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
@@ -52,6 +52,8 @@ import org.apache.tez.runtime.library.common.sort.impl.TezSpillRecord;
import org.apache.tez.runtime.library.common.sort.impl.IFile.Writer;
import org.apache.tez.runtime.library.common.sort.impl.TezMerger.Segment;
+import com.google.common.base.Preconditions;
+
@SuppressWarnings({"unchecked", "rawtypes"})
public class DefaultSorter extends ExternalSorter implements IndexedSortable {
@@ -122,14 +124,9 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable {
TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT,
TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT);
final int sortmb = this.availableMemoryMb;
- if (spillper > (float) 1.0 || spillper <= (float) 0.0) {
- throw new IOException("Invalid \""
- + TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT + "\": " + spillper);
- }
- if ((sortmb & 0x7FF) != sortmb) {
- throw new IOException("Invalid \"" + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB
- + "\": " + sortmb);
- }
+ Preconditions.checkArgument(spillper <= (float) 1.0 && spillper > (float) 0.0,
+ TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT
+ + " should be greater than 0 and less than or equal to 1");
indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT);
http://git-wip-us.apache.org/repos/asf/tez/blob/665801c6/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
index b16936c..afd0090 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/output/TestOnFileSortedOutput.java
@@ -59,6 +59,7 @@ import java.util.Random;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doAnswer;
@@ -158,6 +159,58 @@ public class TestOnFileSortedOutput {
}
@Test
+ public void testSortBufferSize() throws Exception{
+ OutputContext context = createTezOutputContext();
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 2048);
+ UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
+ doReturn(payLoad).when(context).getUserPayload();
+ sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
+ try {
+ sortedOutput.initialize();
+ fail();
+ } catch(IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB));
+ }
+ conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 0);
+ payLoad = TezUtils.createUserPayloadFromConf(conf);
+ doReturn(payLoad).when(context).getUserPayload();
+ sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
+ try {
+ sortedOutput.initialize();
+ fail();
+ } catch(IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB));
+ }
+ }
+
+ @Test
+ public void testSortSpillPercent() throws Exception {
+ OutputContext context = createTezOutputContext();
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 0.0f);
+ UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
+ doReturn(payLoad).when(context).getUserPayload();
+ sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
+ try {
+ sortedOutput.initialize();
+ sortedOutput.start();
+ fail();
+ } catch(IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
+ }
+ conf.setFloat(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT, 1.1f);
+ payLoad = TezUtils.createUserPayloadFromConf(conf);
+ doReturn(payLoad).when(context).getUserPayload();
+ sortedOutput = new OrderedPartitionedKVOutput(context, partitions);
+ try {
+ sortedOutput.initialize();
+ sortedOutput.start();
+ fail();
+ } catch(IllegalArgumentException e) {
+ assertTrue(e.getMessage().contains(TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT));
+ }
+ }
+
+ @Test
public void baseTest() throws Exception {
startSortedOutput(partitions);