You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2015/11/12 20:50:53 UTC
tez git commit: TEZ-2940. Invalid shuffle max slow start setting
causes vertex to hang indefinitely (jeagles)
Repository: tez
Updated Branches:
refs/heads/master 3a1dcb6a3 -> bbcafcb81
TEZ-2940. Invalid shuffle max slow start setting causes vertex to hang indefinitely (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/bbcafcb8
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/bbcafcb8
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/bbcafcb8
Branch: refs/heads/master
Commit: bbcafcb81a548e1e3bca301e9b1d328c97e36384
Parents: 3a1dcb6
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Thu Nov 12 13:50:20 2015 -0600
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Thu Nov 12 13:50:20 2015 -0600
----------------------------------------------------------------------
.../dag/library/vertexmanager/ShuffleVertexManager.java | 6 +++---
.../library/vertexmanager/TestShuffleVertexManager.java | 11 ++++++++++-
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/bbcafcb8/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index d9c4941..c5016aa 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -949,12 +949,12 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION,
ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MAX_SRC_FRACTION_DEFAULT);
- if (slowStartMinSrcCompletionFraction < 0
+ if (slowStartMinSrcCompletionFraction < 0 || slowStartMaxSrcCompletionFraction > 1
|| slowStartMaxSrcCompletionFraction < slowStartMinSrcCompletionFraction) {
throw new IllegalArgumentException(
"Invalid values for slowStartMinSrcCompletionFraction"
- + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0 and "
- + "max cannot be < min.");
+ + "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be > 1,"
+ + " and max cannot be < min.");
}
enableAutoParallelism = conf
http://git-wip-us.apache.org/repos/asf/tez/blob/bbcafcb8/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index 18d4bc1..fec08b2 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -589,7 +589,16 @@ public class TestShuffleVertexManager {
Assert.assertTrue(e.getMessage().contains(
"Invalid values for slowStartMinSrcCompletionFraction"));
}
-
+
+ try {
+ // source vertex have some tasks. max > 1.
+ manager = createManager(conf, mockContext, 0.0f, 95.0f);
+ Assert.assertTrue(false); // should not come here
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Invalid values for slowStartMinSrcCompletionFraction"));
+ }
+
try {
// source vertex have some tasks. min > max
manager = createManager(conf, mockContext, 0.5f, 0.3f);