You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/10 22:07:38 UTC
[02/13] hadoop git commit: YARN-2890. MiniYarnCluster should turn on
timeline service if configured to do so. Contributed by Mit Desai.
YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai.
(cherry picked from commit 265ed1fe804743601a8b62cabc1e4dc2ec8e502f)
(cherry picked from commit 55b794e7fa205df655c19bbfe1de99091fa9dc64)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6b7dc24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6b7dc24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6b7dc24
Branch: refs/heads/branch-2.7
Commit: f6b7dc24f0d2f7e600df95d2f85b6af86d3d64f4
Parents: cfb27d7
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Apr 8 14:13:10 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 10:55:23 2015 -0700
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 52 ++++++++-
hadoop-yarn-project/CHANGES.txt | 4 +-
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 6 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 115 +++++++++++++++++++
6 files changed, 172 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 5d39f35..2b07efb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -496,7 +496,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index c2ef128..eab9026 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,6 +35,52 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+
+ /*
+ * Timeline service should not start if the config is set to false
+ * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
+ cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+
+ @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -42,7 +88,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -96,7 +142,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -133,7 +179,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7ec3457..e144864 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -254,7 +254,6 @@ Release 2.7.1 - 2015-07-06
YARN-3850. NM fails to read files from full disks which can lead to
container logs being lost and other issues (Varun Saxena via jlowe)
-
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
@@ -1039,6 +1038,9 @@ Release 2.6.1 - UNRELEASED
BUG FIXES
+ YARN-2890. MiniYarnCluster should turn on timeline service if
+ configured to do so. (Mit Desai via hitesh)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 5e6fa46..967d172 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -85,7 +85,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, true);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 365e0bb..f8b27b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -262,8 +261,9 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if (enableAHS) {
- addService(new ApplicationHistoryServerWrapper());
+ if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6b7dc24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
new file mode 100644
index 0000000..8a3c9e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestMiniYarnCluster {
+ @Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ int numNodeManagers = 1;
+ int numLocalDirs = 1;
+ int numLogDirs = 1;
+ boolean enableAHS;
+
+ /*
+ * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS flag == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = false;
+ MiniYARNCluster cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == true
+ * and enableAHS == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ enableAHS = false;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS == true
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = true;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+}