You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2015/05/08 00:06:32 UTC
hadoop git commit: MAPREDUCE-6279. AM should explicity exit JVM after
all services have stopped. Contributed by Eric Payne
Repository: hadoop
Updated Branches:
refs/heads/trunk b88700dcd -> f30065c8b
MAPREDUCE-6279. AM should explicity exit JVM after all services have stopped. Contributed by Eric Payne
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f30065c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f30065c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f30065c8
Branch: refs/heads/trunk
Commit: f30065c8b6099372f57015b505434120fe83c2b0
Parents: b88700d
Author: Jason Lowe <jl...@apache.org>
Authored: Thu May 7 22:05:12 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu May 7 22:05:12 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 32 ++++++++++++-
.../mapreduce/v2/app/TestMRAppMaster.java | 49 +++++++++++++++++++-
3 files changed, 81 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f30065c8/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 9af50fa..8f3c960 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -307,6 +307,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6192. Create unit test to automatically compare
MR related classes and mapred-default.xml (rchiang via rkanter)
+ MAPREDUCE-6279. AM should explicity exit JVM after all services have
+ stopped (Eric Payne via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f30065c8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index c41f679..a5c9a25 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -220,6 +220,7 @@ public class MRAppMaster extends CompositeService {
private final ScheduledExecutorService logSyncer;
private long recoveredJobStartTime = 0;
+ private static boolean mainStarted = false;
@VisibleForTesting
protected AtomicBoolean successfullyUnregistered =
@@ -605,11 +606,37 @@ public class MRAppMaster extends CompositeService {
clientService.stop();
} catch (Throwable t) {
LOG.warn("Graceful stop failed. Exiting.. ", t);
- ExitUtil.terminate(1, t);
+ exitMRAppMaster(1, t);
}
+ exitMRAppMaster(0, null);
+ }
+ /** MRAppMaster exit method which has been instrumented for both runtime and
+ * unit testing.
+ * If the main thread has not been started, this method was called from a
+ * test. In that case, configure the ExitUtil object to not exit the JVM.
+ *
+ * @param status integer indicating exit status
+ * @param t throwable exception that could be null
+ */
+ private void exitMRAppMaster(int status, Throwable t) {
+ if (!mainStarted) {
+ ExitUtil.disableSystemExit();
+ }
+ try {
+ if (t != null) {
+ ExitUtil.terminate(status, t);
+ } else {
+ ExitUtil.terminate(status);
+ }
+ } catch (ExitUtil.ExitException ee) {
+ // ExitUtil.ExitException is only thrown from the ExitUtil test code when
+ // SystemExit has been disabled. It is always thrown in in the test code,
+ // even when no error occurs. Ignore the exception so that tests don't
+ // need to handle it.
+ }
}
-
+
private class JobFinishEventHandler implements EventHandler<JobFinishEvent> {
@Override
public void handle(JobFinishEvent event) {
@@ -1407,6 +1434,7 @@ public class MRAppMaster extends CompositeService {
public static void main(String[] args) {
try {
+ mainStarted = true;
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
String containerIdStr =
System.getenv(Environment.CONTAINER_ID.name());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f30065c8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
index 70437c1..63b201d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestMRAppMaster.java
@@ -20,9 +20,15 @@ package org.apache.hadoop.mapreduce.v2.app;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Matchers.anyBoolean;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.times;
+
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -32,7 +38,6 @@ import java.util.HashMap;
import java.util.Map;
import org.junit.Assert;
-
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -63,6 +68,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -437,6 +443,47 @@ public class TestMRAppMaster {
}
+ @Test
+ public void testMRAppMasterShutDownJob() throws Exception,
+ InterruptedException {
+ String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
+ String containerIdStr = "container_1317529182569_0004_000002_1";
+ String userName = "TestAppMasterUser";
+ ApplicationAttemptId applicationAttemptId = ConverterUtils
+ .toApplicationAttemptId(applicationAttemptIdStr);
+ ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
+ JobConf conf = new JobConf();
+ conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
+
+ File stagingDir =
+ new File(MRApps.getStagingAreaDir(conf, userName).toString());
+ stagingDir.mkdirs();
+ MRAppMasterTest appMaster =
+ spy(new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
+ System.currentTimeMillis(), false, true));
+ MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
+ doReturn(conf).when(appMaster).getConfig();
+ appMaster.isLastAMRetry = true;
+ doNothing().when(appMaster).serviceStop();
+ // Test normal shutdown.
+ appMaster.shutDownJob();
+ Assert.assertTrue("Expected shutDownJob to terminate.",
+ ExitUtil.terminateCalled());
+ Assert.assertEquals("Expected shutDownJob to exit with status code of 0.",
+ 0, ExitUtil.getFirstExitException().status);
+
+ // Test shutdown with exception.
+ ExitUtil.resetFirstExitException();
+ String msg = "Injected Exception";
+ doThrow(new RuntimeException(msg))
+ .when(appMaster).notifyIsLastAMRetry(anyBoolean());
+ appMaster.shutDownJob();
+ assertTrue("Expected message from ExitUtil.ExitException to be " + msg,
+ ExitUtil.getFirstExitException().getMessage().contains(msg));
+ Assert.assertEquals("Expected shutDownJob to exit with status code of 1.",
+ 1, ExitUtil.getFirstExitException().status);
+ }
+
private void verifyFailedStatus(MRAppMasterTest appMaster,
String expectedJobState) {
ArgumentCaptor<JobHistoryEvent> captor = ArgumentCaptor