You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2016/06/16 23:08:47 UTC
[1/2] tez git commit: TEZ-3304. TestHistoryParser fails with Hadoop
2.7. (Jonathan Eagles via hitesh)
Repository: tez
Updated Branches:
refs/heads/master 80ba12b2a -> 993a7b5f5
TEZ-3304. TestHistoryParser fails with Hadoop 2.7. (Jonathan Eagles via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/61d5d5e7
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/61d5d5e7
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/61d5d5e7
Branch: refs/heads/master
Commit: 61d5d5e735e38c917a170fa4cc7846d6c38dcdea
Parents: 80ba12b
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Jun 16 16:05:54 2016 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Jun 16 16:05:54 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../java/org/apache/tez/history/TestHistoryParser.java | 10 ++++++----
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/61d5d5e7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 5f90539..0350b48 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3304. TestHistoryParser fails with Hadoop 2.7.
TEZ-3216. Add support for more precise partition stats in VertexManagerEvent.
TEZ-3296. Tez job can hang if two vertices at the same root distance have different task requirements
TEZ-3294. DAG.createDag() does not clear local state on repeat calls.
@@ -62,6 +63,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3304. TestHistoryParser fails with Hadoop 2.7.
TEZ-3296. Tez job can hang if two vertices at the same root distance have different task requirements
TEZ-3294. DAG.createDag() does not clear local state on repeat calls.
TEZ-3297. Deadlock scenario in AM during ShuffleVertexManager auto reduce.
@@ -513,6 +515,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3304. TestHistoryParser fails with Hadoop 2.7.
TEZ-3296. Tez job can hang if two vertices at the same root distance have different task requirements
TEZ-3294. DAG.createDag() does not clear local state on repeat calls.
TEZ-3297. Deadlock scenario in AM during ShuffleVertexManager auto reduce.
http://git-wip-us.apache.org/repos/asf/tez/blob/61d5d5e7/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
index a814425..401a606 100644
--- a/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
+++ b/tez-plugins/tez-history-parser/src/test/java/org/apache/tez/history/TestHistoryParser.java
@@ -117,6 +117,7 @@ public class TestHistoryParser {
private static String TEZ_BASE_DIR =
"target" + Path.SEPARATOR + TestHistoryParser.class.getName() + "-tez";
private static String DOWNLOAD_DIR = TEST_ROOT_DIR + Path.SEPARATOR + "download";
+ private static String yarnTimelineAddress;
@BeforeClass
public static void setupCluster() throws Exception {
@@ -176,10 +177,12 @@ public class TestHistoryParser {
TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "0.0.0.0:8188");
+ tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS));
tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
ATSHistoryLoggingService.class.getName());
+ yarnTimelineAddress = miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
@@ -198,7 +201,7 @@ public class TestHistoryParser {
WordCount.SumProcessor.class.getName(), "WordCount", true);
//Export the data from ATS
- String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR };
+ String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
int result = ATSImportTool.process(args);
assertTrue(result == 0);
@@ -363,7 +366,7 @@ public class TestHistoryParser {
.getName(), "WordCount-With-Exception", true);
//Export the data from ATS
- String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR };
+ String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
int result = ATSImportTool.process(args);
assertTrue(result == 0);
@@ -611,7 +614,6 @@ public class TestHistoryParser {
TezConfiguration tezConf = new TezConfiguration(miniTezCluster.getConfig());
if (withTimeline) {
tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, withTimeline);
- tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "0.0.0.0:8188");
tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
ATSHistoryLoggingService.class.getName());
} else {
[2/2] tez git commit: TEZ-3305. TestAnalyzer fails on Hadoop 2.7.
(Jonathan Eagles via hitesh)
Posted by hi...@apache.org.
TEZ-3305. TestAnalyzer fails on Hadoop 2.7. (Jonathan Eagles via hitesh)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/993a7b5f
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/993a7b5f
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/993a7b5f
Branch: refs/heads/master
Commit: 993a7b5f58223f811d82f2a1dec5ed7349922b2e
Parents: 61d5d5e
Author: Hitesh Shah <hi...@apache.org>
Authored: Thu Jun 16 16:07:47 2016 -0700
Committer: Hitesh Shah <hi...@apache.org>
Committed: Thu Jun 16 16:07:47 2016 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../src/test/java/org/apache/tez/analyzer/TestAnalyzer.java | 7 +++++--
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/993a7b5f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 0350b48..d060d17 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3305. TestAnalyzer fails on Hadoop 2.7.
TEZ-3304. TestHistoryParser fails with Hadoop 2.7.
TEZ-3216. Add support for more precise partition stats in VertexManagerEvent.
TEZ-3296. Tez job can hang if two vertices at the same root distance have different task requirements
@@ -63,6 +64,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3305. TestAnalyzer fails on Hadoop 2.7.
TEZ-3304. TestHistoryParser fails with Hadoop 2.7.
TEZ-3296. Tez job can hang if two vertices at the same root distance have different task requirements
TEZ-3294. DAG.createDag() does not clear local state on repeat calls.
@@ -515,6 +517,7 @@ INCOMPATIBLE CHANGES
ALL CHANGES:
+ TEZ-3305. TestAnalyzer fails on Hadoop 2.7.
TEZ-3304. TestHistoryParser fails with Hadoop 2.7.
TEZ-3296. Tez job can hang if two vertices at the same root distance have different task requirements
TEZ-3294. DAG.createDag() does not clear local state on repeat calls.
http://git-wip-us.apache.org/repos/asf/tez/blob/993a7b5f/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java
----------------------------------------------------------------------
diff --git a/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java b/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java
index f680f59..d34f9c5 100644
--- a/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java
+++ b/tez-tools/analyzers/job-analyzer/src/test/java/org/apache/tez/analyzer/TestAnalyzer.java
@@ -88,6 +88,7 @@ public class TestAnalyzer {
private boolean usingATS = true;
private boolean downloadedSimpleHistoryFile = false;
+ private static String yarnTimelineAddress;
@BeforeClass
public static void setupClass() throws Exception {
@@ -137,6 +138,7 @@ public class TestAnalyzer {
miniTezCluster.init(conf);
miniTezCluster.start();
+ yarnTimelineAddress = miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
private TezConfiguration createCommonTezLog() throws Exception {
@@ -156,7 +158,8 @@ public class TestAnalyzer {
private void createTezSessionATS() throws Exception {
TezConfiguration tezConf = createCommonTezLog();
tezConf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, "0.0.0.0:8188");
+ tezConf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ miniTezCluster.getConfig().get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS));
tezConf.setBoolean(TezConfiguration.TEZ_AM_ALLOW_DISABLED_TIMELINE_DOMAINS, true);
tezConf.set(TezConfiguration.TEZ_HISTORY_LOGGING_SERVICE_CLASS,
ATSHistoryLoggingService.class.getName());
@@ -255,7 +258,7 @@ public class TestAnalyzer {
DagInfo dagInfo = null;
if (usingATS) {
//Export the data from ATS
- String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR };
+ String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
int result = ATSImportTool.process(args);
assertTrue(result == 0);