You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2015/12/11 17:17:55 UTC
tez git commit: TEZ-2966. Tez does not honor mapreduce.task.timeout
(jeagles)
Repository: tez
Updated Branches:
refs/heads/master b03faf1ad -> 460e0b2a0
TEZ-2966. Tez does not honor mapreduce.task.timeout (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/460e0b2a
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/460e0b2a
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/460e0b2a
Branch: refs/heads/master
Commit: 460e0b2a01c00748bb80ad2314b2168dfa1e9615
Parents: b03faf1
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Fri Dec 11 10:16:25 2015 -0600
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Fri Dec 11 10:16:25 2015 -0600
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../apache/tez/mapreduce/hadoop/DeprecatedKeys.java | 3 ++-
.../tez/mapreduce/hadoop/TestDeprecatedKeys.java | 13 ++++++++++---
3 files changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/460e0b2a/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 60fb502..ac8af72 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
TEZ-2949. Allow duplicate dag names within session for Tez.
ALL CHANGES:
+ TEZ-2966. Tez does not honor mapreduce.task.timeout
TEZ-2979. FlakyTest: org.apache.tez.history.TestHistoryParser.
TEZ-1491. Tez reducer-side merge's counter update is slow.
TEZ-2943. Change shuffle vertex manager to use per vertex data for auto
@@ -278,6 +279,7 @@ INCOMPATIBLE CHANGES
TEZ-2949. Allow duplicate dag names within session for Tez.
ALL CHANGES
+ TEZ-2966. Tez does not honor mapreduce.task.timeout
TEZ-2346. TEZ-UI: Lazy load other info / counter data
TEZ-2975. Bump up apache commons dependency.
TEZ-2970. Re-localization in TezChild does not use correct UGI.
http://git-wip-us.apache.org/repos/asf/tez/blob/460e0b2a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
index 49f95c0..345f7aa 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/DeprecatedKeys.java
@@ -84,7 +84,7 @@ public class DeprecatedKeys {
TezConfiguration.TEZ_COUNTERS_GROUP_NAME_MAX_LENGTH);
}
- // TODO TEZAM4 Sometime, make sure this gets loaded by default. Insteaf of the current initialization in MRAppMaster, TezChild.
+ // TODO TEZAM4 Sometime, make sure this gets loaded by default. Instead of the current initialization in MRAppMaster, TezChild.
// Maybe define in an TEZConfiguration / TEZ JobConf variant.
public static void init() {
@@ -154,6 +154,7 @@ public class DeprecatedKeys {
registerMRToRuntimeKeyTranslation(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS_CODEC);
+ registerMRToRuntimeKeyTranslation(MRJobConfig.TASK_TIMEOUT, TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS);
}
private static void addDeprecatedKeys() {
http://git-wip-us.apache.org/repos/asf/tez/blob/460e0b2a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
index 2414743..06d2990 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/hadoop/TestDeprecatedKeys.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
import org.apache.tez.runtime.library.api.TezRuntimeConfiguration;
import org.apache.tez.runtime.library.common.Constants;
@@ -42,6 +43,7 @@ public class TestDeprecatedKeys {
jobConf.setFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT, 0.22f);
jobConf.setBoolean(MRJobConfig.REDUCE_MEMTOMEM_ENABLED, true);
jobConf.setFloat(MRJobConfig.REDUCE_INPUT_BUFFER_PERCENT, 0.33f);
+ jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000);
MRHelpers.translateMRConfToTez(jobConf);
@@ -63,11 +65,12 @@ public class TestDeprecatedKeys {
assertEquals(0.33f,
jobConf.getFloat(TezRuntimeConfiguration.TEZ_RUNTIME_INPUT_POST_MERGE_BUFFER_PERCENT, 0),
0.01f);
+ assertEquals(1000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000));
}
@Test(timeout = 5000)
/**
- * Set of keys that can be overriden at tez runtime
+ * Set of keys that can be overridden at tez runtime
*/
public void verifyTezOverridenKeys() {
JobConf jobConf = new JobConf();
@@ -75,7 +78,8 @@ public class TestDeprecatedKeys {
jobConf.setInt(MRJobConfig.IO_SORT_MB, 100);
jobConf.setInt(MRJobConfig.COUNTERS_MAX_KEY, 100);
jobConf.setFloat(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART, 0.95f);
-
+ jobConf.setInt(MRJobConfig.TASK_TIMEOUT, 1000);
+
jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_FACTOR, 1000);
jobConf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 200);
jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_IFILE_READAHEAD, true);
@@ -99,7 +103,8 @@ public class TestDeprecatedKeys {
jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, "DefaultSorter");
jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_GROUP_COMPARATOR_CLASS, "groupComparator");
jobConf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_SECONDARY_COMPARATOR_CLASS, "SecondaryComparator");
-
+ jobConf.setInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000);
+
jobConf.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
jobConf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, true);
@@ -130,6 +135,7 @@ public class TestDeprecatedKeys {
assertEquals("DefaultSorter", jobConf.get(TezRuntimeConfiguration.TEZ_RUNTIME_INTERNAL_SORTER_CLASS, ""));
assertTrue(jobConf.getBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_COMPRESS, false));
assertEquals(0.95f, jobConf.getFloat(ShuffleVertexManager.TEZ_SHUFFLE_VERTEX_MANAGER_MIN_SRC_FRACTION, 0.0f), 0.0f);
+ assertEquals(2000, jobConf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 2000));
assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD));
assertNull(jobConf.get(MRConfig.MAPRED_IFILE_READAHEAD_BYTES));
@@ -155,6 +161,7 @@ public class TestDeprecatedKeys {
assertNull(jobConf.get(MRJobConfig.GROUP_COMPARATOR_CLASS));
assertNull(jobConf.get("map.sort.class"));
assertNull(jobConf.get(MRJobConfig.COMPLETED_MAPS_FOR_REDUCE_SLOWSTART));
+ assertNull(jobConf.get(MRJobConfig.TASK_TIMEOUT));
}
}