You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2019/09/09 10:21:03 UTC
[hadoop] branch branch-3.2.1 updated (865b49a -> b3cbbb4)
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a change to branch branch-3.2.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from 865b49a Preparing for 3.2.1 release.
new 46a1a6f YARN-9820. RM logs InvalidStateTransitionException when app is submitted. Contributed by Prabhu Joseph
new b3cbbb4 YARN-9821. NM hangs at serviceStop when ATSV2 Backend Hbase is Down. Contributed by Prabhu Joseph.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../resourcemanager/recovery/RMStateStore.java | 7 ++
.../recovery/RMStateUpdateAppEvent.java | 7 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 +-
.../rmapp/TestRMAppTransitions.java | 7 +-
.../storage/HBaseTimelineWriterImpl.java | 74 ++++++++++++----------
5 files changed, 62 insertions(+), 35 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: YARN-9820. RM logs InvalidStateTransitionException
when app is submitted. Contributed by Prabhu Joseph
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch branch-3.2.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 46a1a6fc7d5e18f0caec48ce33ecb21901adfac8
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Mon Sep 9 00:23:04 2019 -0700
YARN-9820. RM logs InvalidStateTransitionException when app is submitted. Contributed by Prabhu Joseph
---
.../hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java | 7 +++++++
.../server/resourcemanager/recovery/RMStateUpdateAppEvent.java | 7 ++++++-
.../apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 2 +-
.../yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java | 7 ++++++-
4 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index ccd6fc9..01606bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -868,6 +868,13 @@ public abstract class RMStateStore extends AbstractService {
getRMStateStoreEventHandler().handle(new RMStateUpdateAppEvent(appState));
}
+ @SuppressWarnings("unchecked")
+ public void updateApplicationState(ApplicationStateData appState,
+ boolean notifyApp) {
+ getRMStateStoreEventHandler().handle(new RMStateUpdateAppEvent(appState,
+ notifyApp));
+ }
+
public void updateApplicationStateSynchronously(ApplicationStateData appState,
boolean notifyApp, SettableFuture<Object> resultFuture) {
handleStoreEvent(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
index 0a6220b..a43d43e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateUpdateAppEvent.java
@@ -30,9 +30,14 @@ public class RMStateUpdateAppEvent extends RMStateStoreEvent {
private SettableFuture<Object> future;
public RMStateUpdateAppEvent(ApplicationStateData appState) {
+ this (appState, true);
+ }
+
+ public RMStateUpdateAppEvent(ApplicationStateData appState,
+ boolean notifyApplication) {
super(RMStateStoreEventType.UPDATE_APP);
this.appState = appState;
- this.notifyApplication = true;
+ this.notifyApplication = notifyApplication;
this.future = null;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index a1e4558..84c1d86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1037,7 +1037,7 @@ public class RMAppImpl implements RMApp, Recoverable {
app.callerContext);
appState.setApplicationTimeouts(app.getApplicationTimeouts());
appState.setLaunchTime(event.getTimestamp());
- app.rmContext.getStateStore().updateApplicationState(appState);
+ app.rmContext.getStateStore().updateApplicationState(appState, false);
app.launchTime = event.getTimestamp();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index f5dcefb..b32ad1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
@@ -472,8 +473,12 @@ public class TestRMAppTransitions {
private void assertAppStateLaunchTimeSaved(long expectedLaunchTime) {
ArgumentCaptor<ApplicationStateData> state =
ArgumentCaptor.forClass(ApplicationStateData.class);
- verify(store, times(1)).updateApplicationState(state.capture());
+ ArgumentCaptor<Boolean> notifyApp =
+ ArgumentCaptor.forClass(Boolean.class);
+ verify(store, times(1)).updateApplicationState(state.capture(),
+ notifyApp.capture());
assertEquals(expectedLaunchTime, state.getValue().getLaunchTime());
+ assertFalse(notifyApp.getValue());
}
private void assertKilled(RMApp application) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: YARN-9821. NM hangs at serviceStop when ATSV2
Backend Hbase is Down. Contributed by Prabhu Joseph.
Posted by ro...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch branch-3.2.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit b3cbbb467e22ea829b3808f4b7b01d07e0bf3842
Author: Abhishek Modi <ab...@apache.org>
AuthorDate: Mon Sep 9 14:26:00 2019 +0530
YARN-9821. NM hangs at serviceStop when ATSV2 Backend Hbase is Down. Contributed by Prabhu Joseph.
---
.../storage/HBaseTimelineWriterImpl.java | 74 ++++++++++++----------
1 file changed, 42 insertions(+), 32 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index a398feb..dda004d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice-hbase/hadoop-yarn-server-timelineservice-hbase-client/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -630,39 +630,49 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
*/
@Override
protected void serviceStop() throws Exception {
- if (entityTable != null) {
- LOG.info("closing the entity table");
- // The close API performs flushing and releases any resources held
- entityTable.close();
+ boolean isStorageUp = true;
+ try {
+ storageMonitor.checkStorageIsUp();
+ } catch (IOException e) {
+ LOG.warn("Failed to close the timeline tables as Hbase is down", e);
+ isStorageUp = false;
}
- if (appToFlowTable != null) {
- LOG.info("closing the app_flow table");
- // The close API performs flushing and releases any resources held
- appToFlowTable.close();
- }
- if (applicationTable != null) {
- LOG.info("closing the application table");
- applicationTable.close();
- }
- if (flowRunTable != null) {
- LOG.info("closing the flow run table");
- // The close API performs flushing and releases any resources held
- flowRunTable.close();
- }
- if (flowActivityTable != null) {
- LOG.info("closing the flowActivityTable table");
- // The close API performs flushing and releases any resources held
- flowActivityTable.close();
- }
- if (subApplicationTable != null) {
- subApplicationTable.close();
- }
- if (domainTable != null) {
- domainTable.close();
- }
- if (conn != null) {
- LOG.info("closing the hbase Connection");
- conn.close();
+
+ if (isStorageUp) {
+ if (entityTable != null) {
+ LOG.info("closing the entity table");
+ // The close API performs flushing and releases any resources held
+ entityTable.close();
+ }
+ if (appToFlowTable != null) {
+ LOG.info("closing the app_flow table");
+ // The close API performs flushing and releases any resources held
+ appToFlowTable.close();
+ }
+ if (applicationTable != null) {
+ LOG.info("closing the application table");
+ applicationTable.close();
+ }
+ if (flowRunTable != null) {
+ LOG.info("closing the flow run table");
+ // The close API performs flushing and releases any resources held
+ flowRunTable.close();
+ }
+ if (flowActivityTable != null) {
+ LOG.info("closing the flowActivityTable table");
+ // The close API performs flushing and releases any resources held
+ flowActivityTable.close();
+ }
+ if (subApplicationTable != null) {
+ subApplicationTable.close();
+ }
+ if (domainTable != null) {
+ domainTable.close();
+ }
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
}
storageMonitor.stop();
super.serviceStop();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org