You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2015/05/22 23:35:34 UTC
hadoop git commit: YARN-3701. Isolating the error of generating a
single app report when getting all apps from generic history service.
Contributed by Zhijie Shen
Repository: hadoop
Updated Branches:
refs/heads/trunk f34638399 -> 455b3acf0
YARN-3701. Isolating the error of generating a single app report when
getting all apps from generic history service. Contributed by Zhijie
Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/455b3acf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/455b3acf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/455b3acf
Branch: refs/heads/trunk
Commit: 455b3acf0e82b214e06bd7b538968252945cd3c4
Parents: f346383
Author: Xuan <xg...@apache.org>
Authored: Fri May 22 14:34:39 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Fri May 22 14:34:39 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
...pplicationHistoryManagerOnTimelineStore.java | 37 +++++++++++++++-----
...pplicationHistoryManagerOnTimelineStore.java | 25 +++++++++----
3 files changed, 50 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/455b3acf/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c98bfe3..d7edbf2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -550,6 +550,9 @@ Release 2.7.1 - UNRELEASED
YARN-3675. FairScheduler: RM quits when node removal races with
continuous-scheduling on the same node. (Anubhav Dhoot via kasha)
+ YARN-3701. Isolating the error of generating a single app report when
+ getting all apps from generic history service. (Zhijie Shen via xgong)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/455b3acf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index db00d2c..2429a68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -25,6 +25,8 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -67,6 +69,8 @@ import com.google.common.annotations.VisibleForTesting;
public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
implements
ApplicationHistoryManager {
+ private static final Log LOG = LogFactory
+ .getLog(ApplicationHistoryManagerOnTimelineStore.class);
@VisibleForTesting
static final String UNAVAILABLE = "N/A";
@@ -107,9 +111,14 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
for (TimelineEntity entity : entities.getEntities()) {
- ApplicationReportExt app =
- generateApplicationReport(entity, ApplicationReportField.ALL);
- apps.put(app.appReport.getApplicationId(), app.appReport);
+ try {
+ ApplicationReportExt app =
+ generateApplicationReport(entity, ApplicationReportField.ALL);
+ apps.put(app.appReport.getApplicationId(), app.appReport);
+ } catch (Exception e) {
+ LOG.error("Error on generating application report for " +
+ entity.getEntityId(), e);
+ }
}
}
return apps;
@@ -142,9 +151,18 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
@Override
public ApplicationAttemptReport getApplicationAttempt(
ApplicationAttemptId appAttemptId) throws YarnException, IOException {
- ApplicationReportExt app = getApplication(
- appAttemptId.getApplicationId(), ApplicationReportField.USER_AND_ACLS);
- checkAccess(app);
+ return getApplicationAttempt(appAttemptId, true);
+ }
+
+ private ApplicationAttemptReport getApplicationAttempt(
+ ApplicationAttemptId appAttemptId, boolean checkACLs)
+ throws YarnException, IOException {
+ if (checkACLs) {
+ ApplicationReportExt app = getApplication(
+ appAttemptId.getApplicationId(),
+ ApplicationReportField.USER_AND_ACLS);
+ checkAccess(app);
+ }
TimelineEntity entity = timelineDataManager.getEntity(
AppAttemptMetricsConstants.ENTITY_TYPE,
appAttemptId.toString(), EnumSet.allOf(Field.class),
@@ -182,7 +200,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
@Override
public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
throws YarnException, IOException {
- ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId);
+ ApplicationAttemptReport appAttempt =
+ getApplicationAttempt(appAttemptId, false);
return getContainer(appAttempt.getAMContainerId());
}
@@ -517,8 +536,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
try {
checkAccess(app);
if (app.appReport.getCurrentApplicationAttemptId() != null) {
- ApplicationAttemptReport appAttempt =
- getApplicationAttempt(app.appReport.getCurrentApplicationAttemptId());
+ ApplicationAttemptReport appAttempt = getApplicationAttempt(
+ app.appReport.getCurrentApplicationAttemptId(), false);
app.appReport.setHost(appAttempt.getHost());
app.appReport.setRpcPort(appAttempt.getRpcPort());
app.appReport.setTrackingUrl(appAttempt.getTrackingUrl());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/455b3acf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 8cf1240..fcdafc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -78,7 +78,9 @@ public class TestApplicationHistoryManagerOnTimelineStore {
store = createStore(SCALE);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(createApplicationTimelineEntity(
- ApplicationId.newInstance(0, SCALE + 1), true, false));
+ ApplicationId.newInstance(0, SCALE + 1), true, true, false));
+ entities.addEntity(createApplicationTimelineEntity(
+ ApplicationId.newInstance(0, SCALE + 2), true, false, true));
store.put(entities);
}
@@ -133,9 +135,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
TimelineEntities entities = new TimelineEntities();
ApplicationId appId = ApplicationId.newInstance(0, i);
if (i == 2) {
- entities.addEntity(createApplicationTimelineEntity(appId, true, true));
+ entities.addEntity(createApplicationTimelineEntity(
+ appId, true, false, false));
} else {
- entities.addEntity(createApplicationTimelineEntity(appId, false, true));
+ entities.addEntity(createApplicationTimelineEntity(
+ appId, false, false, false));
}
store.put(entities);
for (int j = 1; j <= scale; ++j) {
@@ -334,6 +338,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
historyManager.getAllApplications().values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
+ ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
+ for (ApplicationReport app : apps) {
+ Assert.assertNotEquals(ignoredAppId, app.getApplicationId());
+ }
}
@Test
@@ -433,10 +441,15 @@ public class TestApplicationHistoryManagerOnTimelineStore {
}
private static TimelineEntity createApplicationTimelineEntity(
- ApplicationId appId, boolean emptyACLs, boolean noAttempt) {
+ ApplicationId appId, boolean emptyACLs, boolean noAttemptId,
+ boolean wrongAppId) {
TimelineEntity entity = new TimelineEntity();
entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(appId.toString());
+ if (wrongAppId) {
+ entity.setEntityId("wrong_app_id");
+ } else {
+ entity.setEntityId(appId.toString());
+ }
entity.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
entity.addPrimaryFilter(
TimelineStore.SystemFilter.ENTITY_OWNER.toString(), "yarn");
@@ -472,7 +485,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
FinalApplicationStatus.UNDEFINED.toString());
eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
YarnApplicationState.FINISHED.toString());
- if (noAttempt) {
+ if (!noAttemptId) {
eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
ApplicationAttemptId.newInstance(appId, 1));
}