You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/05/02 19:22:02 UTC
[27/50] [abbrv] hadoop git commit: YARN-7786. NullPointerException
while launching ApplicationMaster. Contributed by lujie
YARN-7786. NullPointerException while launching ApplicationMaster. Contributed by lujie
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06f3f453
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06f3f453
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06f3f453
Branch: refs/heads/YARN-8200
Commit: 06f3f453c22dfc77c1444ab58606bfb30282f671
Parents: 733248c
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Apr 20 13:31:19 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Apr 20 13:31:19 2018 -0500
----------------------------------------------------------------------
.../resourcemanager/amlauncher/AMLauncher.java | 21 +++--
.../TestApplicationMasterLauncher.java | 82 ++++++++++++++++----
2 files changed, 84 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f3f453/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index e0754a0..19f1d81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -105,7 +105,7 @@ public class AMLauncher implements Runnable {
connect();
ContainerId masterContainerID = masterContainer.getId();
ApplicationSubmissionContext applicationContext =
- application.getSubmissionContext();
+ application.getSubmissionContext();
LOG.info("Setting up container " + masterContainer
+ " for AM " + application.getAppAttemptId());
ContainerLaunchContext launchContext =
@@ -189,6 +189,10 @@ public class AMLauncher implements Runnable {
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
+ if (container == null){
+ throw new IOException(containerID +
+ " has been cleaned before launched");
+ }
// Finalize the container
setupTokens(container, containerID);
// set the flow context optionally for timeline service v.2
@@ -308,11 +312,7 @@ public class AMLauncher implements Runnable {
handler.handle(new RMAppAttemptEvent(application.getAppAttemptId(),
RMAppAttemptEventType.LAUNCHED));
} catch(Exception ie) {
- String message = "Error launching " + application.getAppAttemptId()
- + ". Got exception: " + StringUtils.stringifyException(ie);
- LOG.info(message);
- handler.handle(new RMAppAttemptEvent(application
- .getAppAttemptId(), RMAppAttemptEventType.LAUNCH_FAILED, message));
+ onAMLaunchFailed(masterContainer.getId(), ie);
}
break;
case CLEANUP:
@@ -347,4 +347,13 @@ public class AMLauncher implements Runnable {
throw (IOException) t;
}
}
+
+ @SuppressWarnings("unchecked")
+ protected void onAMLaunchFailed(ContainerId containerId, Exception ie) {
+ String message = "Error launching " + application.getAppAttemptId()
+ + ". Got exception: " + StringUtils.stringifyException(ie);
+ LOG.info(message);
+ handler.handle(new RMAppAttemptEvent(application
+ .getAppAttemptId(), RMAppAttemptEventType.LAUNCH_FAILED, message));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06f3f453/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
index 1603c2d..d834474 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
@@ -24,12 +24,14 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.TimeoutException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -73,6 +75,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.utils.AMRMClientUtils;
@@ -83,6 +86,9 @@ import org.apache.log4j.Logger;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.base.Supplier;
+
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -206,7 +212,7 @@ public class TestApplicationMasterLauncher {
public void testAMLaunchAndCleanup() throws Exception {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
- MyContainerManagerImpl containerManager = new MyContainerManagerImpl();
+ final MyContainerManagerImpl containerManager = new MyContainerManagerImpl();
MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(
containerManager);
rm.start();
@@ -217,10 +223,14 @@ public class TestApplicationMasterLauncher {
// kick the scheduling
nm1.nodeHeartbeat(true);
- int waitCount = 0;
- while (containerManager.launched == false && waitCount++ < 20) {
- LOG.info("Waiting for AM Launch to happen..");
- Thread.sleep(1000);
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return containerManager.launched;
+ }
+ }, 100, 200 * 100);
+ } catch (TimeoutException e) {
+ fail("timed out while waiting for AM Launch to happen.");
}
Assert.assertTrue(containerManager.launched);
@@ -234,7 +244,7 @@ public class TestApplicationMasterLauncher {
.getMasterContainer().getId()
.toString(), containerManager.containerIdAtContainerManager);
Assert.assertEquals(nm1.getNodeId().toString(),
- containerManager.nmHostAtContainerManager);
+ containerManager.nmHostAtContainerManager);
Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
containerManager.maxAppAttempts);
@@ -247,10 +257,14 @@ public class TestApplicationMasterLauncher {
nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
- waitCount = 0;
- while (containerManager.cleanedup == false && waitCount++ < 20) {
- LOG.info("Waiting for AM Cleanup to happen..");
- Thread.sleep(1000);
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return containerManager.cleanedup;
+ }
+ }, 100, 200 * 100);
+ } catch (TimeoutException e) {
+ fail("timed out while waiting for AM cleanup to happen.");
}
Assert.assertTrue(containerManager.cleanedup);
@@ -259,6 +273,48 @@ public class TestApplicationMasterLauncher {
}
@Test
+ public void testAMCleanupBeforeLaunch() throws Exception {
+ MockRM rm = new MockRM();
+ rm.start();
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);
+ RMApp app = rm.submitApp(2000);
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+ final RMAppAttempt attempt = app.getCurrentAppAttempt();
+
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override public Boolean get() {
+ return attempt.getMasterContainer() != null;
+ }
+ }, 10, 200 * 100);
+ } catch (TimeoutException e) {
+ fail("timed out while waiting for AM Launch to happen.");
+ }
+
+ //send kill before launch
+ rm.killApp(app.getApplicationId());
+ rm.waitForState(app.getApplicationId(), RMAppState.KILLED);
+ //Launch after kill
+ AMLauncher launcher = new AMLauncher(rm.getRMContext(),
+ attempt, AMLauncherEventType.LAUNCH, rm.getConfig()) {
+ @Override
+ public void onAMLaunchFailed(ContainerId containerId, Exception e) {
+ Assert.assertFalse("NullPointerException happens "
+ + " while launching " + containerId,
+ e instanceof NullPointerException);
+ }
+ @Override
+ protected ContainerManagementProtocol getContainerMgrProxy(
+ ContainerId containerId) {
+ return new MyContainerManagerImpl();
+ }
+ };
+ launcher.run();
+ rm.stop();
+ }
+
+ @Test
public void testRetriesOnFailures() throws Exception {
final ContainerManagementProtocol mockProxy =
mock(ContainerManagementProtocol.class);
@@ -304,7 +360,7 @@ public class TestApplicationMasterLauncher {
rm.drainEvents();
MockRM.waitForState(app.getCurrentAppAttempt(),
- RMAppAttemptState.LAUNCHED, 500);
+ RMAppAttemptState.LAUNCHED, 500);
}
@SuppressWarnings("unused")
@@ -336,9 +392,9 @@ public class TestApplicationMasterLauncher {
AllocateResponse amrs = null;
try {
- amrs = am.allocate(new ArrayList<ResourceRequest>(),
+ amrs = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>());
- Assert.fail();
+ Assert.fail();
} catch (ApplicationMasterNotRegisteredException e) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org