You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2014/10/01 01:44:27 UTC
git commit: YARN-2602. Fixed possible NPE in
ApplicationHistoryManagerOnTimelineStore. Contributed by Zhijie Shen
Repository: hadoop
Updated Branches:
refs/heads/trunk d7075ada5 -> bbff96be4
YARN-2602. Fixed possible NPE in ApplicationHistoryManagerOnTimelineStore. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbff96be
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbff96be
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbff96be
Branch: refs/heads/trunk
Commit: bbff96be48119774688981d04baf444639135977
Parents: d7075ad
Author: Jian He <ji...@apache.org>
Authored: Tue Sep 30 16:39:25 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Tue Sep 30 16:44:17 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
...pplicationHistoryManagerOnTimelineStore.java | 4 +-
...pplicationHistoryManagerOnTimelineStore.java | 110 ++++++++------
.../metrics/SystemMetricsPublisher.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 146 ++++++++++---------
5 files changed, 149 insertions(+), 116 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff96be/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 95fba23..bfaaa90 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -491,6 +491,9 @@ Release 2.6.0 - UNRELEASED
YARN-2594. Potential deadlock in RM when querying
ApplicationResourceUsageReport. (Wangda Tan via kasha)
+ YARN-2602. Fixed possible NPE in ApplicationHistoryManagerOnTimelineStore.
+ (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff96be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index f00ec9c..5381bd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -227,7 +227,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO)) {
String appViewACLsStr = entityInfo.get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO).toString();
- appViewACLs.put(ApplicationAccessType.VIEW_APP, appViewACLsStr);
+ if (appViewACLsStr.length() > 0) {
+ appViewACLs.put(ApplicationAccessType.VIEW_APP, appViewACLsStr);
+ }
}
if (field == ApplicationReportField.USER_AND_ACLS) {
return new ApplicationReportExt(ApplicationReport.newInstance(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff96be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index e6bfcd9..49386c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -122,7 +122,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
for (int i = 1; i <= SCALE; ++i) {
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.newInstance(0, i);
- entities.addEntity(createApplicationTimelineEntity(appId));
+ if (i == 2) {
+ entities.addEntity(createApplicationTimelineEntity(appId, true));
+ } else {
+ entities.addEntity(createApplicationTimelineEntity(appId, false));
+ }
store.put(entities);
for (int j = 1; j <= SCALE; ++j) {
entities = new TimelineEntities();
@@ -142,50 +146,58 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplicationReport() throws Exception {
- final ApplicationId appId = ApplicationId.newInstance(0, 1);
- ApplicationReport app;
- if (callerUGI == null) {
- app = historyManager.getApplication(appId);
- } else {
- app =
- callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
- @Override
- public ApplicationReport run() throws Exception {
- return historyManager.getApplication(appId);
- }
- });
- }
- Assert.assertNotNull(app);
- Assert.assertEquals(appId, app.getApplicationId());
- Assert.assertEquals("test app", app.getName());
- Assert.assertEquals("test app type", app.getApplicationType());
- Assert.assertEquals("user1", app.getUser());
- Assert.assertEquals("test queue", app.getQueue());
- Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
- Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
- Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
- if (callerUGI != null && callerUGI.getShortUserName().equals("user3")) {
- Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
- app.getCurrentApplicationAttemptId());
- Assert.assertEquals(null, app.getHost());
- Assert.assertEquals(-1, app.getRpcPort());
- Assert.assertEquals(null, app.getTrackingUrl());
- Assert.assertEquals(null, app.getOriginalTrackingUrl());
- Assert.assertEquals(null, app.getDiagnostics());
- } else {
- Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1),
- app.getCurrentApplicationAttemptId());
- Assert.assertEquals("test host", app.getHost());
- Assert.assertEquals(-100, app.getRpcPort());
- Assert.assertEquals("test tracking url", app.getTrackingUrl());
- Assert.assertEquals("test original tracking url",
- app.getOriginalTrackingUrl());
- Assert.assertEquals("test diagnostics info", app.getDiagnostics());
+ for (int i = 1; i <= 2; ++i) {
+ final ApplicationId appId = ApplicationId.newInstance(0, i);
+ ApplicationReport app;
+ if (callerUGI == null) {
+ app = historyManager.getApplication(appId);
+ } else {
+ app =
+ callerUGI.doAs(new PrivilegedExceptionAction<ApplicationReport> () {
+ @Override
+ public ApplicationReport run() throws Exception {
+ return historyManager.getApplication(appId);
+ }
+ });
+ }
+ Assert.assertNotNull(app);
+ Assert.assertEquals(appId, app.getApplicationId());
+ Assert.assertEquals("test app", app.getName());
+ Assert.assertEquals("test app type", app.getApplicationType());
+ Assert.assertEquals("user1", app.getUser());
+ Assert.assertEquals("test queue", app.getQueue());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+ // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
+ // Nobody except admin and owner has access to the details of the app.
+ if ((i == 1 && callerUGI != null &&
+ callerUGI.getShortUserName().equals("user3")) ||
+ (i == 2 && callerUGI != null &&
+ (callerUGI.getShortUserName().equals("user2") ||
+ callerUGI.getShortUserName().equals("user3")))) {
+ Assert.assertEquals(ApplicationAttemptId.newInstance(appId, -1),
+ app.getCurrentApplicationAttemptId());
+ Assert.assertEquals(null, app.getHost());
+ Assert.assertEquals(-1, app.getRpcPort());
+ Assert.assertEquals(null, app.getTrackingUrl());
+ Assert.assertEquals(null, app.getOriginalTrackingUrl());
+ Assert.assertEquals(null, app.getDiagnostics());
+ } else {
+ Assert.assertEquals(ApplicationAttemptId.newInstance(appId, 1),
+ app.getCurrentApplicationAttemptId());
+ Assert.assertEquals("test host", app.getHost());
+ Assert.assertEquals(-100, app.getRpcPort());
+ Assert.assertEquals("test tracking url", app.getTrackingUrl());
+ Assert.assertEquals("test original tracking url",
+ app.getOriginalTrackingUrl());
+ Assert.assertEquals("test diagnostics info", app.getDiagnostics());
+ }
+ Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
+ app.getFinalApplicationStatus());
+ Assert.assertEquals(YarnApplicationState.FINISHED,
+ app.getYarnApplicationState());
}
- Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
- app.getFinalApplicationStatus());
- Assert.assertEquals(YarnApplicationState.FINISHED,
- app.getYarnApplicationState());
}
@Test
@@ -396,7 +408,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}
private static TimelineEntity createApplicationTimelineEntity(
- ApplicationId appId) {
+ ApplicationId appId, boolean emptyACLs) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
entity.setEntityId(appId.toString());
@@ -410,8 +422,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
- entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
- "user2");
+ if (emptyACLs) {
+ entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
+ } else {
+ entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
+ "user2");
+ }
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff96be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 5da006c..e2ecf9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -137,7 +137,7 @@ public class SystemMetricsPublisher extends CompositeService {
dispatcher.getEventHandler().handle(
new ApplicationACLsUpdatedEvent(
app.getApplicationId(),
- appViewACLs,
+ appViewACLs == null ? "" : appViewACLs,
updatedTime));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbff96be/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 63343e9..52faf12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -95,77 +95,89 @@ public class TestSystemMetricsPublisher {
@Test(timeout = 10000)
public void testPublishApplicationMetrics() throws Exception {
- ApplicationId appId = ApplicationId.newInstance(0, 1);
- RMApp app = createRMApp(appId);
- metricsPublisher.appCreated(app, app.getStartTime());
- metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
- metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
- TimelineEntity entity = null;
- do {
- entity =
- store.getEntity(appId.toString(),
- ApplicationMetricsConstants.ENTITY_TYPE,
- EnumSet.allOf(Field.class));
- // ensure three events are both published before leaving the loop
- } while (entity == null || entity.getEvents().size() < 3);
- // verify all the fields
- Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE,
- entity.getEntityType());
- Assert
- .assertEquals(app.getApplicationId().toString(), entity.getEntityId());
- Assert
- .assertEquals(
- app.getName(),
- entity.getOtherInfo().get(
- ApplicationMetricsConstants.NAME_ENTITY_INFO));
- Assert.assertEquals(app.getQueue(),
- entity.getOtherInfo()
- .get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
- Assert
- .assertEquals(
- app.getUser(),
- entity.getOtherInfo().get(
- ApplicationMetricsConstants.USER_ENTITY_INFO));
- Assert
- .assertEquals(
- app.getApplicationType(),
+ for (int i = 1; i <= 2; ++i) {
+ ApplicationId appId = ApplicationId.newInstance(0, i);
+ RMApp app = createRMApp(appId);
+ metricsPublisher.appCreated(app, app.getStartTime());
+ metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
+ if (i == 1) {
+ metricsPublisher.appACLsUpdated(app, "uers1,user2", 4L);
+ } else {
+ // in case user doesn't specify the ACLs
+ metricsPublisher.appACLsUpdated(app, null, 4L);
+ }
+ TimelineEntity entity = null;
+ do {
+ entity =
+ store.getEntity(appId.toString(),
+ ApplicationMetricsConstants.ENTITY_TYPE,
+ EnumSet.allOf(Field.class));
+ // ensure three events are both published before leaving the loop
+ } while (entity == null || entity.getEvents().size() < 3);
+ // verify all the fields
+ Assert.assertEquals(ApplicationMetricsConstants.ENTITY_TYPE,
+ entity.getEntityType());
+ Assert
+ .assertEquals(app.getApplicationId().toString(), entity.getEntityId());
+ Assert
+ .assertEquals(
+ app.getName(),
+ entity.getOtherInfo().get(
+ ApplicationMetricsConstants.NAME_ENTITY_INFO));
+ Assert.assertEquals(app.getQueue(),
+ entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO));
+ Assert
+ .assertEquals(
+ app.getUser(),
+ entity.getOtherInfo().get(
+ ApplicationMetricsConstants.USER_ENTITY_INFO));
+ Assert
+ .assertEquals(
+ app.getApplicationType(),
+ entity.getOtherInfo().get(
+ ApplicationMetricsConstants.TYPE_ENTITY_INFO));
+ Assert.assertEquals(app.getSubmitTime(),
+ entity.getOtherInfo().get(
+ ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
+ if (i == 1) {
+ Assert.assertEquals("uers1,user2",
entity.getOtherInfo().get(
- ApplicationMetricsConstants.TYPE_ENTITY_INFO));
- Assert.assertEquals(app.getSubmitTime(),
- entity.getOtherInfo().get(
- ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO));
- Assert.assertEquals("uers1,user2",
- entity.getOtherInfo().get(
+ ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
+ } else {
+ Assert.assertEquals("", entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
- boolean hasCreatedEvent = false;
- boolean hasFinishedEvent = false;
- boolean hasACLsUpdatedEvent = false;
- for (TimelineEvent event : entity.getEvents()) {
- if (event.getEventType().equals(
- ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
- hasCreatedEvent = true;
- Assert.assertEquals(app.getStartTime(), event.getTimestamp());
- } else if (event.getEventType().equals(
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
- hasFinishedEvent = true;
- Assert.assertEquals(app.getFinishTime(), event.getTimestamp());
- Assert.assertEquals(
- app.getDiagnostics().toString(),
- event.getEventInfo().get(
- ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
- Assert.assertEquals(
- app.getFinalApplicationStatus().toString(),
- event.getEventInfo().get(
- ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
- Assert.assertEquals(YarnApplicationState.FINISHED.toString(), event
- .getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
- } else if (event.getEventType().equals(
- ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
- hasACLsUpdatedEvent = true;
- Assert.assertEquals(4L, event.getTimestamp());
}
+ boolean hasCreatedEvent = false;
+ boolean hasFinishedEvent = false;
+ boolean hasACLsUpdatedEvent = false;
+ for (TimelineEvent event : entity.getEvents()) {
+ if (event.getEventType().equals(
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
+ hasCreatedEvent = true;
+ Assert.assertEquals(app.getStartTime(), event.getTimestamp());
+ } else if (event.getEventType().equals(
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ hasFinishedEvent = true;
+ Assert.assertEquals(app.getFinishTime(), event.getTimestamp());
+ Assert.assertEquals(
+ app.getDiagnostics().toString(),
+ event.getEventInfo().get(
+ ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO));
+ Assert.assertEquals(
+ app.getFinalApplicationStatus().toString(),
+ event.getEventInfo().get(
+ ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO));
+ Assert.assertEquals(YarnApplicationState.FINISHED.toString(), event
+ .getEventInfo().get(ApplicationMetricsConstants.STATE_EVENT_INFO));
+ } else if (event.getEventType().equals(
+ ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE)) {
+ hasACLsUpdatedEvent = true;
+ Assert.assertEquals(4L, event.getTimestamp());
+ }
+ }
+ Assert.assertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent);
}
- Assert.assertTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent);
}
@Test(timeout = 10000)