You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2014/12/02 04:08:01 UTC
hadoop git commit: YARN-2890. MiniYARNCluster should start the
timeline server based on the configuration. Contributed by Mit Desai.
Repository: hadoop
Updated Branches:
refs/heads/trunk a7fba0bc2 -> 51af8d367
YARN-2890. MiniYARNCluster should start the timeline server based on the configuration. Contributed by Mit Desai.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51af8d36
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51af8d36
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51af8d36
Branch: refs/heads/trunk
Commit: 51af8d367de94689770f57c64bea3b244d7755f6
Parents: a7fba0b
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Dec 1 19:04:04 2014 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Dec 1 19:04:04 2014 -0800
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 58 +++++++++++++++++++-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 6 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 24 +-------
7 files changed, 67 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index de35d84..a64f1d6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 346953f..76a3598 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,6 +35,58 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+
+ /*
+ * Timeline service should not start if the config is set to false
+ * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+
+ //Run a MR job and verify it succeeds
+ Path inDir = new Path("input");
+ Path outDir = new Path("output");
+ RunningJob job =
+ UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
+ Assert.assertEquals(JobStatus.SUCCEEDED,
+ job.getJobStatus().getState().getValue());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
+ cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+
+ @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -42,7 +94,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -96,7 +148,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -133,7 +185,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 47b38a1..45b736e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -72,11 +72,7 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
}
public MiniMRYarnCluster(String testName, int noOfNMs) {
- this(testName, noOfNMs, false);
- }
-
- public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) {
- super(testName, 1, noOfNMs, 4, 4, enableAHS);
+ super(testName, 1, noOfNMs, 4, 4);
historyServerWrapper = new JobHistoryServerWrapper();
addService(historyServerWrapper);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6a7d55d..9e0bbe3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -160,6 +160,9 @@ Release 2.7.0 - UNRELEASED
YARN-2905. AggregatedLogsBlock page can infinitely loop if the aggregated
log file is corrupted (Varun Saxena via jlowe)
+ YARN-2890. MiniYARNCluster should start the timeline server based on the
+ configuration. (Mit Desai via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 1d3a104..a05b3b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -84,7 +84,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, true);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index da7d505..4b2b4bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -295,7 +295,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
- numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
+ numOfNMs, 1, 1, overrideClientRMService, overrideRTS,
overrideApplicationMasterService);
cluster.resetStartFailoverFlag(false);
cluster.init(conf);
@@ -326,10 +326,10 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public MiniYARNClusterForHATesting(String testName,
int numResourceManagers, int numNodeManagers, int numLocalDirs,
- int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
+ int numLogDirs, boolean overrideClientRMService,
boolean overrideRTS, boolean overrideApplicationMasterService) {
super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
- numLogDirs, enableAHS);
+ numLogDirs);
this.overrideClientRMService = overrideClientRMService;
this.overrideRTS = overrideRTS;
this.overrideApplicationMasterService = overrideApplicationMasterService;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51af8d36/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index e83d601..2eb5034 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -120,7 +119,6 @@ public class MiniYARNCluster extends CompositeService {
private int numLocalDirs;
// Number of nm-log-dirs per nodemanager
private int numLogDirs;
- private boolean enableAHS;
/**
* @param testName name of the test
@@ -128,15 +126,13 @@ public class MiniYARNCluster extends CompositeService {
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
- * @param enableAHS enable ApplicationHistoryServer or not
*/
public MiniYARNCluster(
String testName, int numResourceManagers, int numNodeManagers,
- int numLocalDirs, int numLogDirs, boolean enableAHS) {
+ int numLocalDirs, int numLogDirs) {
super(testName.replace("$", ""));
this.numLocalDirs = numLocalDirs;
this.numLogDirs = numLogDirs;
- this.enableAHS = enableAHS;
String testSubDir = testName.replace("$", "");
File targetWorkDir = new File("target", testSubDir);
try {
@@ -188,20 +184,6 @@ public class MiniYARNCluster extends CompositeService {
/**
* @param testName name of the test
- * @param numResourceManagers the number of resource managers in the cluster
- * @param numNodeManagers the number of node managers in the cluster
- * @param numLocalDirs the number of nm-local-dirs per nodemanager
- * @param numLogDirs the number of nm-log-dirs per nodemanager
- */
- public MiniYARNCluster(
- String testName, int numResourceManagers, int numNodeManagers,
- int numLocalDirs, int numLogDirs) {
- this(testName, numResourceManagers, numNodeManagers, numLocalDirs,
- numLogDirs, false);
- }
-
- /**
- * @param testName name of the test
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
@@ -260,8 +242,8 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if (enableAHS) {
- addService(new ApplicationHistoryServerWrapper());
+ if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(