You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2015/11/21 08:56:13 UTC
[09/50] [abbrv] hadoop git commit: YARN-4347. Resource manager fails
with Null pointer exception. (Jian He via wangda)
YARN-4347. Resource manager fails with Null pointer exception. (Jian He via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f55a180
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f55a180
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f55a180
Branch: refs/heads/HDFS-7240
Commit: 7f55a18071ebe1c5591ff8a410b227c855c3f428
Parents: 796638d
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 12 11:23:40 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Nov 12 11:23:40 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 26 +++++++++++--
.../TestWorkPreservingRMRestart.java | 41 ++++++++++++++++++++
4 files changed, 67 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f55a180/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 52c6669..aaa222d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1065,6 +1065,8 @@ Release 2.7.3 - UNRELEASED
BUG FIXES
+ YARN-4347. Resource manager fails with Null pointer exception. (Jian He via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f55a180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 41254d8..bcfce9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1382,7 +1382,7 @@ public class RMAppImpl implements RMApp, Recoverable {
|| appState == RMAppState.KILLED;
}
- private RMAppState getRecoveredFinalState() {
+ public RMAppState getRecoveredFinalState() {
return this.recoveredFinalState;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f55a180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 48144c3..88a89b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent;
@@ -1102,6 +1103,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
@Override
public RMAppAttemptState transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
+ RMApp rmApp = appAttempt.rmContext.getRMApps().get(
+ appAttempt.getAppAttemptId().getApplicationId());
+
/*
* If last attempt recovered final state is null .. it means attempt was
* started but AM container may or may not have started / finished.
@@ -1109,8 +1113,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
*/
if (appAttempt.recoveredFinalState != null) {
appAttempt.progress = 1.0f;
- RMApp rmApp =appAttempt.rmContext.getRMApps().get(
- appAttempt.getAppAttemptId().getApplicationId());
if (appAttempt.submissionContext
.getKeepContainersAcrossApplicationAttempts()
@@ -1130,7 +1132,24 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt, event);
}
return appAttempt.recoveredFinalState;
- } else {
+ } else if (RMAppImpl.isAppInFinalState(rmApp)) {
+ // Somehow attempt final state was not saved but app final state was saved.
+ // Skip adding the attempt into scheduler
+ RMAppState appState = ((RMAppImpl) rmApp).getRecoveredFinalState();
+ LOG.warn(rmApp.getApplicationId() + " final state (" + appState
+ + ") was recorded, but " + appAttempt.applicationAttemptId
+ + " final state (" + appAttempt.recoveredFinalState
+ + ") was not recorded.");
+ switch (appState) {
+ case FINISHED:
+ return RMAppAttemptState.FINISHED;
+ case FAILED:
+ return RMAppAttemptState.FAILED;
+ case KILLED:
+ return RMAppAttemptState.KILLED;
+ }
+ return RMAppAttemptState.FAILED;
+ } else{
// Add the current attempt to the scheduler.
if (appAttempt.rmContext.isWorkPreservingRecoveryEnabled()) {
// Need to register an app attempt before AM can register
@@ -1164,6 +1183,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
+
private void rememberTargetTransitions(RMAppAttemptEvent event,
Object transitionToDo, RMAppAttemptState targetFinalState) {
transitionTodo = transitionToDo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f55a180/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 2ef427e..0571387 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.TestRMRestart.TestSecurityMockRM;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -1367,4 +1368,44 @@ public class TestWorkPreservingRMRestart extends ParameterizedSchedulerTestBase
assertEquals(1,loadedAttempt1.getJustFinishedContainers().size());
}
+ // Test that if application state was saved, but attempt state was not saved.
+ // RM should start correctly.
+ @Test (timeout = 20000)
+ public void testAppStateSavedButAttemptStateNotSaved() throws Exception {
+ MemoryRMStateStore memStore = new MemoryRMStateStore() {
+ @Override public synchronized void updateApplicationAttemptStateInternal(
+ ApplicationAttemptId appAttemptId,
+ ApplicationAttemptStateData attemptState) {
+ // do nothing;
+ // simulate the failure that attempt final state is not saved.
+ }
+ };
+ memStore.init(conf);
+ rm1 = new MockRM(conf, memStore);
+ rm1.start();
+
+ MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ RMApp app1 = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
+
+ ApplicationStateData appSavedState =
+ memStore.getState().getApplicationState().get(app1.getApplicationId());
+
+ // check that app state is saved.
+ assertEquals(RMAppState.FINISHED, appSavedState.getState());
+ // check that attempt state is not saved.
+ assertNull(appSavedState.getAttempt(am1.getApplicationAttemptId()).getState());
+
+ rm2 = new MockRM(conf, memStore);
+ rm2.start();
+ RMApp recoveredApp1 =
+ rm2.getRMContext().getRMApps().get(app1.getApplicationId());
+
+ assertEquals(RMAppState.FINISHED, recoveredApp1.getState());
+ // check that attempt state is recovered correctly.
+ assertEquals(RMAppAttemptState.FINISHED, recoveredApp1.getCurrentAppAttempt().getState());
+ }
}