You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2017/05/16 16:10:23 UTC
[06/50] [abbrv] hadoop git commit: YARN-6598. History server
getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via
jeagles)
YARN-6598. History server getApplicationReport NPE when fetching report for pre-2.8 job (Jason Lowe via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c48f2976
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c48f2976
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c48f2976
Branch: refs/heads/YARN-2915
Commit: c48f2976a3de60b95c4a5ada4f0131c4cdde177a
Parents: 6600abb
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Mon May 15 10:32:01 2017 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Mon May 15 10:32:01 2017 -0500
----------------------------------------------------------------------
...pplicationHistoryManagerOnTimelineStore.java | 35 ++++++++++++--------
...pplicationHistoryManagerOnTimelineStore.java | 32 ++++++++++++++----
2 files changed, 48 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c48f2976/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 54893ef..d18f3dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -330,20 +330,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
- long vcoreSeconds=Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants.APP_CPU_METRICS).toString());
- long memorySeconds=Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants.APP_MEM_METRICS).toString());
- long preemptedMemorySeconds = Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants
- .APP_MEM_PREEMPT_METRICS).toString());
- long preemptedVcoreSeconds = Long.parseLong(entityInfo.get(
- ApplicationMetricsConstants
- .APP_CPU_PREEMPT_METRICS).toString());
- appResources = ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0,
- 0, preemptedMemorySeconds, preemptedVcoreSeconds);
+ long vcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_METRICS);
+ long memorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_METRICS);
+ long preemptedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
+ long preemptedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
+ appResources = ApplicationResourceUsageReport.newInstance(0, 0, null,
+ null, null, memorySeconds, vcoreSeconds, 0, 0,
+ preemptedMemorySeconds, preemptedVcoreSeconds);
}
+
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>();
Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
@@ -445,6 +444,16 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static long parseLong(Map<String, Object> entityInfo,
+ String infoKey) {
+ long result = 0;
+ Object infoValue = entityInfo.get(infoKey);
+ if (infoValue != null) {
+ result = Long.parseLong(infoValue.toString());
+ }
+ return result;
+ }
+
private static boolean isFinalState(YarnApplicationState state) {
return state == YarnApplicationState.FINISHED
|| state == YarnApplicationState.FAILED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c48f2976/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 3d4e91b..9600251 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -143,6 +143,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
if (i == 2) {
entities.addEntity(createApplicationTimelineEntity(
appId, true, false, false, true, YarnApplicationState.FINISHED));
+ } else if (i == 3) {
+ entities.addEntity(createApplicationTimelineEntity(
+ appId, false, false, false, false, YarnApplicationState.FINISHED,
+ true));
} else {
entities.addEntity(createApplicationTimelineEntity(
appId, false, false, false, false, YarnApplicationState.FINISHED));
@@ -176,7 +180,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplicationReport() throws Exception {
- for (int i = 1; i <= 2; ++i) {
+ for (int i = 1; i <= 3; ++i) {
final ApplicationId appId = ApplicationId.newInstance(0, i);
ApplicationReport app;
if (callerUGI == null) {
@@ -214,7 +218,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
- if ((i == 1 && callerUGI != null &&
+ if ((i != 2 && callerUGI != null &&
callerUGI.getShortUserName().equals("user3")) ||
(i == 2 && callerUGI != null &&
(callerUGI.getShortUserName().equals("user2") ||
@@ -245,10 +249,16 @@ public class TestApplicationHistoryManagerOnTimelineStore {
applicationResourceUsageReport.getMemorySeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
- Assert.assertEquals(456,
+ long expectedPreemptMemSecs = 456;
+ long expectedPreemptVcoreSecs = 789;
+ if (i == 3) {
+ expectedPreemptMemSecs = 0;
+ expectedPreemptVcoreSecs = 0;
+ }
+ Assert.assertEquals(expectedPreemptMemSecs,
applicationResourceUsageReport.getPreemptedMemorySeconds());
Assert
- .assertEquals(789, applicationResourceUsageReport
+ .assertEquals(expectedPreemptVcoreSecs, applicationResourceUsageReport
.getPreemptedVcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus());
@@ -486,6 +496,14 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
boolean wrongAppId, boolean enableUpdateEvent,
YarnApplicationState state) {
+ return createApplicationTimelineEntity(appId, emptyACLs, noAttemptId,
+ wrongAppId, enableUpdateEvent, state, false);
+ }
+
+ private static TimelineEntity createApplicationTimelineEntity(
+ ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
+ boolean wrongAppId, boolean enableUpdateEvent,
+ YarnApplicationState state, boolean missingPreemptMetrics) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
if (wrongAppId) {
@@ -510,8 +528,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Integer.MAX_VALUE + 1L);
entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789);
+ if (!missingPreemptMetrics) {
+ entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
+ entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
+ }
if (emptyACLs) {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org