You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/08/27 22:54:34 UTC
[hadoop] branch branch-3.2 updated: YARN-9438. launchTime not
written to state store for running applications
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new f36ccf0 YARN-9438. launchTime not written to state store for running applications
f36ccf0 is described below
commit f36ccf0ac1465a336b9165cffaec024d973a6033
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Tue Aug 27 15:35:17 2019 -0700
YARN-9438. launchTime not written to state store for running applications
(cherry picked from commit 9568656cd21d9c02168e18ce35c6726077bbf3a1)
---
.../yarn/server/resourcemanager/RMAppManager.java | 2 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 6 ++++++
.../scheduler/capacity/CapacityScheduler.java | 1 +
.../yarn/server/resourcemanager/TestRMRestart.java | 19 +++++++++++--------
.../rmapp/TestRMAppTransitions.java | 22 ++++++++++++++++++++++
5 files changed, 42 insertions(+), 8 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index c7dcacf..2783f36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -695,6 +695,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
app.getStartTime(), app.getApplicationSubmissionContext(),
app.getUser(), app.getCallerContext());
appState.setApplicationTimeouts(currentExpireTimeouts);
+ appState.setLaunchTime(app.getLaunchTime());
// update to state store. Though it synchronous call, update via future to
// know any exception has been set. It is required because in non-HA mode,
@@ -820,6 +821,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
app.getApplicationSubmissionContext(), app.getUser(),
app.getCallerContext());
appState.setApplicationTimeouts(app.getApplicationTimeouts());
+ appState.setLaunchTime(app.getLaunchTime());
rmContext.getStateStore().updateApplicationStateSynchronously(appState,
false, future);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 9d32257..a1e4558 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1032,6 +1032,12 @@ public class RMAppImpl implements RMApp, Recoverable {
app.getApplicationId()+", attemptId: "+
app.getCurrentAppAttempt().getAppAttemptId()+
"launchTime: "+event.getTimestamp());
+ ApplicationStateData appState = ApplicationStateData.newInstance(
+ app.submitTime, app.startTime, app.submissionContext, app.user,
+ app.callerContext);
+ appState.setApplicationTimeouts(app.getApplicationTimeouts());
+ appState.setLaunchTime(event.getTimestamp());
+ app.rmContext.getStateStore().updateApplicationState(appState);
app.launchTime = event.getTimestamp();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 45efd4a..9b3a453 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2704,6 +2704,7 @@ public class CapacityScheduler extends
rmApp.getApplicationSubmissionContext(), rmApp.getUser(),
rmApp.getCallerContext());
appState.setApplicationTimeouts(rmApp.getApplicationTimeouts());
+ appState.setLaunchTime(rmApp.getLaunchTime());
rmContext.getStateStore().updateApplicationStateSynchronously(appState,
false, future);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 03dd04a..5f6624c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -754,13 +754,14 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
@Override
public void updateApplicationStateInternal(ApplicationId appId,
ApplicationStateData appStateData) throws Exception {
- if (count == 0) {
- // do nothing; simulate app final state is not saved.
+ if (count == 1) {
+ // Application state is updated on attempt launch.
+ // After that, do nothing; simulate app final state is not saved.
LOG.info(appId + " final state is not saved.");
- count++;
} else {
super.updateApplicationStateInternal(appId, appStateData);
}
+ count++;
}
};
memStore.init(conf);
@@ -774,7 +775,6 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120);
RMApp app0 = rm1.submitApp(200);
MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);
-
FinishApplicationMasterRequest req =
FinishApplicationMasterRequest.newInstance(
FinalApplicationStatus.SUCCEEDED, "", "");
@@ -1797,8 +1797,11 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED);
rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
- Assert.assertEquals(1, ((TestMemoryRMStateStore) memStore).updateAttempt);
- Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateApp);
+ // count = 1 on storing RMApp launchTime
+ // count = 2 on storing attempt state on kill
+ // count = 3 on storing app state on kill
+ Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateAttempt);
+ Assert.assertEquals(3, ((TestMemoryRMStateStore) memStore).updateApp);
}
// Test Application that fails on submission is saved in state store.
@@ -2547,8 +2550,6 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
// create an app and finish the app.
RMApp app0 = rm1.submitApp(200);
- ApplicationStateData app0State = memStore.getState().getApplicationState()
- .get(app0.getApplicationId());
MockAM am0 = launchAndFailAM(app0, rm1, nm1);
MockAM am1 = launchAndFailAM(app0, rm1, nm1);
@@ -2557,6 +2558,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
// am1 is missed from MemoryRMStateStore
memStore.removeApplicationAttemptInternal(am1.getApplicationAttemptId());
+ ApplicationStateData app0State = memStore.getState().getApplicationState()
+ .get(app0.getApplicationId());
ApplicationAttemptStateData am2State = app0State.getAttempt(
am2.getApplicationAttemptId());
// am2's state is not consistent: MemoryRMStateStore just saved its initial
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 3eb6ef4..f5dcefb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -469,6 +469,13 @@ public class TestRMAppTransitions {
any(ApplicationStateData.class));
}
+ private void assertAppStateLaunchTimeSaved(long expectedLaunchTime) {
+ ArgumentCaptor<ApplicationStateData> state =
+ ArgumentCaptor.forClass(ApplicationStateData.class);
+ verify(store, times(1)).updateApplicationState(state.capture());
+ assertEquals(expectedLaunchTime, state.getValue().getLaunchTime());
+ }
+
private void assertKilled(RMApp application) {
assertTimesAtFinish(application);
assertAppState(RMAppState.KILLED, application);
@@ -903,6 +910,21 @@ public class TestRMAppTransitions {
}
@Test
+ public void testAppAcceptedAccepted() throws IOException {
+ LOG.info("--- START: testAppAcceptedAccepted ---");
+
+ RMApp application = testCreateAppAccepted(null);
+ // ACCEPTED => ACCEPTED event RMAppEventType.ATTEMPT_LAUNCHED
+ RMAppEvent appAttemptLaunched =
+ new RMAppEvent(application.getApplicationId(),
+ RMAppEventType.ATTEMPT_LAUNCHED, 1234L);
+ application.handle(appAttemptLaunched);
+ rmDispatcher.await();
+ assertAppState(RMAppState.ACCEPTED, application);
+ assertAppStateLaunchTimeSaved(1234L);
+ }
+
+ @Test
public void testAppAcceptedAttemptKilled() throws IOException,
InterruptedException {
LOG.info("--- START: testAppAcceptedAttemptKilled ---");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org