You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by sr...@apache.org on 2016/02/24 19:51:37 UTC
[3/9] tez git commit: TEZ-3131. Support a way to override
test_root_dir for FaultToleranceTestRunner. (hitesh)
TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner. (hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/44ca2295
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/44ca2295
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/44ca2295
Branch: refs/heads/TEZ-2980
Commit: 44ca2295f8209dd83041e426d0dca95721461de6
Parents: f38e23c
Author: Hitesh Shah <hi...@apache.org>
Authored: Mon Feb 22 17:13:16 2016 -0800
Committer: Hitesh Shah <hi...@apache.org>
Committed: Mon Feb 22 17:13:16 2016 -0800
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../library/vertexmanager/ShuffleVertexManager.java | 4 +++-
.../apache/tez/test/FaultToleranceTestRunner.java | 15 ++++++++++++---
3 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index dc27f93..2311c55 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
TEZ-3029. Add an onError method to service plugin contexts.
ALL CHANGES:
+ TEZ-3131. Support a way to override test_root_dir for FaultToleranceTestRunner.
TEZ-3067. Links to tez configs documentation should be bubbled up to top-level release page.
TEZ-3123. Containers can get re-used even with conflicting local resources.
TEZ-3117. Deadlock in Edge and Vertex code
http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index 410ad73..ea00532 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -1004,7 +1004,9 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
throw new IllegalArgumentException(
"Invalid values for slowStartMinSrcCompletionFraction"
+ "/slowStartMaxSrcCompletionFraction. Min cannot be < 0, max cannot be > 1,"
- + " and max cannot be < min.");
+ + " and max cannot be < min."
+ + ", configuredMin=" + slowStartMinSrcCompletionFraction
+ + ", configuredMax=" + slowStartMaxSrcCompletionFraction);
}
enableAutoParallelism = conf
http://git-wip-us.apache.org/repos/asf/tez/blob/44ca2295/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
index 84e5b97..94242a7 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
@@ -44,7 +44,8 @@ import org.apache.hadoop.util.GenericOptionsParser;
*/
public class FaultToleranceTestRunner {
- static String TEST_ROOT_DIR = "tmp";
+ static String DEFAULT_FT_STAGING_DIR = "tmp";
+ static String FT_STAGING_DIR = "tez.test-fault-tolerance.staging-dir";
Configuration conf = null;
TezClient tezSession = null;
Resource defaultResource = Resource.newInstance(100, 0);
@@ -57,8 +58,16 @@ public class FaultToleranceTestRunner {
tezConf = new TezConfiguration(new YarnConfiguration(this.conf));
}
FileSystem defaultFs = FileSystem.get(tezConf);
-
- Path remoteStagingDir = defaultFs.makeQualified(new Path(TEST_ROOT_DIR, String
+
+ Path testRootDir;
+ if (tezConf.get(FT_STAGING_DIR) != null
+ && !tezConf.get(FT_STAGING_DIR).isEmpty()) {
+ testRootDir = new Path(tezConf.get(FT_STAGING_DIR));
+ } else {
+ testRootDir = new Path(DEFAULT_FT_STAGING_DIR);
+ }
+
+ Path remoteStagingDir = defaultFs.makeQualified(new Path(testRootDir, String
.valueOf(new Random().nextInt(100000))));
TezClientUtils.ensureStagingDirExists(tezConf, remoteStagingDir);