You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2015/12/16 19:52:23 UTC
hadoop git commit: YARN-4452. NPE when submit Unmanaged application.
Contributed by Naganarasimha G R. (cherry picked from commit
50bd067e1d63d4c80dc1e7bf4024bfaf42cf4416)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 65cbac9de -> 1a2ef845b
YARN-4452. NPE when submit Unmanaged application. Contributed by Naganarasimha G R.
(cherry picked from commit 50bd067e1d63d4c80dc1e7bf4024bfaf42cf4416)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a2ef845
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a2ef845
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a2ef845
Branch: refs/heads/branch-2
Commit: 1a2ef845b54166194b133e524ad9533cc259aed2
Parents: 65cbac9
Author: Junping Du <ju...@apache.org>
Authored: Wed Dec 16 10:57:39 2015 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Wed Dec 16 10:58:40 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 9 ++++++
.../metrics/SystemMetricsPublisher.java | 22 ++++++++-----
.../metrics/TestSystemMetricsPublisher.java | 34 ++++++++++++++++----
3 files changed, 51 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a2ef845/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 22e4518..502163b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1086,6 +1086,9 @@ Release 2.8.0 - UNRELEASED
YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the
lastScheduler time. (Lin Yiqun via zxu)
+ YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R
+ via junping_du)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1134,6 +1137,9 @@ Release 2.7.3 - UNRELEASED
YARN-4439. Clarify NMContainerStatus#toString method. (Jian He via xgong)
+ YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R via
+ junping_du)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -1994,6 +2000,9 @@ Release 2.6.4 - UNRELEASED
YARN-3535. Scheduler must re-request container resources when RMContainer transitions
from ALLOCATED to KILLED (rohithsharma and peng.zhang via asuresh)
+ YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R
+ via junping_du)
+
Release 2.6.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a2ef845/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 44cb807..f240660 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -160,6 +160,8 @@ public class SystemMetricsPublisher extends CompositeService {
public void appAttemptRegistered(RMAppAttempt appAttempt,
long registeredTime) {
if (publishSystemMetrics) {
+ ContainerId container = (appAttempt.getMasterContainer() == null) ? null
+ : appAttempt.getMasterContainer().getId();
dispatcher.getEventHandler().handle(
new AppAttemptRegisteredEvent(
appAttempt.getAppAttemptId(),
@@ -167,7 +169,7 @@ public class SystemMetricsPublisher extends CompositeService {
appAttempt.getRpcPort(),
appAttempt.getTrackingUrl(),
appAttempt.getOriginalTrackingUrl(),
- appAttempt.getMasterContainer().getId(),
+ container,
registeredTime));
}
}
@@ -176,6 +178,8 @@ public class SystemMetricsPublisher extends CompositeService {
public void appAttemptFinished(RMAppAttempt appAttempt,
RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
if (publishSystemMetrics) {
+ ContainerId container = (appAttempt.getMasterContainer() == null) ? null
+ : appAttempt.getMasterContainer().getId();
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
appAttempt.getAppAttemptId(),
@@ -187,7 +191,7 @@ public class SystemMetricsPublisher extends CompositeService {
app.getFinalApplicationStatus(),
RMServerUtils.createApplicationAttemptState(appAttemtpState),
finishedTime,
- appAttempt.getMasterContainer().getId()));
+ container));
}
}
@@ -390,9 +394,10 @@ public class SystemMetricsPublisher extends CompositeService {
event.getHost());
eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
event.getRpcPort());
- eventInfo.put(
- AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
- event.getMasterContainerId().toString());
+ if (event.getMasterContainerId() != null) {
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ event.getMasterContainerId().toString());
+ }
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);
@@ -417,9 +422,10 @@ public class SystemMetricsPublisher extends CompositeService {
event.getFinalApplicationStatus().toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
event.getYarnApplicationAttemptState().toString());
- eventInfo.put(
- AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
- event.getMasterContainerId().toString());
+ if (event.getMasterContainerId() != null) {
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ event.getMasterContainerId().toString());
+ }
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a2ef845/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 4835ade..f2b02bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -257,10 +257,30 @@ public class TestSystemMetricsPublisher {
}
@Test(timeout = 10000)
+ public void testPublishAppAttemptMetricsForUnmanagedAM() throws Exception {
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
+ RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId,true);
+ metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
+ RMApp app = mock(RMApp.class);
+ when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
+ metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED, app,
+ Integer.MAX_VALUE + 2L);
+ TimelineEntity entity = null;
+ do {
+ entity =
+ store.getEntity(appAttemptId.toString(),
+ AppAttemptMetricsConstants.ENTITY_TYPE,
+ EnumSet.allOf(Field.class));
+ // ensure two events are both published before leaving the loop
+ } while (entity == null || entity.getEvents().size() < 2);
+ }
+
+ @Test(timeout = 10000)
public void testPublishAppAttemptMetrics() throws Exception {
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
- RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId);
+ RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId, false);
metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
RMApp app = mock(RMApp.class);
when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);
@@ -435,15 +455,17 @@ public class TestSystemMetricsPublisher {
}
private static RMAppAttempt createRMAppAttempt(
- ApplicationAttemptId appAttemptId) {
+ ApplicationAttemptId appAttemptId, boolean unmanagedAMAttempt) {
RMAppAttempt appAttempt = mock(RMAppAttempt.class);
when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
when(appAttempt.getHost()).thenReturn("test host");
when(appAttempt.getRpcPort()).thenReturn(-100);
- Container container = mock(Container.class);
- when(container.getId())
- .thenReturn(ContainerId.newContainerId(appAttemptId, 1));
- when(appAttempt.getMasterContainer()).thenReturn(container);
+ if (!unmanagedAMAttempt) {
+ Container container = mock(Container.class);
+ when(container.getId())
+ .thenReturn(ContainerId.newContainerId(appAttemptId, 1));
+ when(appAttempt.getMasterContainer()).thenReturn(container);
+ }
when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
when(appAttempt.getOriginalTrackingUrl()).thenReturn(