You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/10 22:07:43 UTC
[07/13] hadoop git commit: YARN-3700. Made generic history service
load a number of latest applications according to the parameter or the
configuration. Contributed by Xuan Gong.
YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.
(cherry picked from commit 54504133f41e36eaea6bb06c7b9ddb249468ecd7)
(cherry picked from commit 839f81a6326b2f8b3d5183178382c1551b0bc259)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e48bffd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e48bffd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e48bffd0
Branch: refs/heads/branch-2.7
Commit: e48bffd05598ff86ff7d185640f13a134470c300
Parents: d3562b4
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed May 27 16:51:48 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Thu Sep 10 11:56:27 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../src/main/resources/yarn-default.xml | 9 +++++++
.../ApplicationHistoryClientService.java | 2 +-
.../ApplicationHistoryManager.java | 12 ++++++---
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 13 +++++++---
.../TestApplicationHistoryClientService.java | 26 +++++++++++++++++---
...pplicationHistoryManagerOnTimelineStore.java | 12 +++++----
.../hadoop/yarn/server/webapp/AppsBlock.java | 7 ++++++
.../hadoop/yarn/server/webapp/WebServices.java | 17 ++++---------
.../src/site/markdown/TimelineServer.md | 1 +
13 files changed, 84 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e144864..9bd5cb1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -26,6 +26,9 @@ Release 2.7.2 - UNRELEASED
YARN-2801. Add documentation for node labels feature. (Wangda Tan and Naganarasimha
G R via ozawa)
+ YARN-3700. Made generic history service load a number of latest applications
+ according to the parameter or the configuration. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7b71be6..c0517ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1381,6 +1381,15 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+ /**
+ * Defines the max number of applications could be fetched using
+ * REST API or application history protocol and shown in timeline
+ * server web ui.
+ */
+ public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ APPLICATION_HISTORY_PREFIX + "max-applications";
+ public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 405d319..bb4be32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
+ String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index cb3ad97..de6906e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1348,6 +1348,15 @@
</property>
<property>
+ <description>
+ Defines the max number of applications could be fetched using REST API or
+ application history protocol and shown in timeline server web ui.
+ </description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+ </property>
+
+ <property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index e64ca14..7427926 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -186,7 +186,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
IOException {
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
- history.getAllApplications().values()));
+ history.getApplications(request.getLimit()).values()));
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..c218eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -51,16 +51,20 @@ public interface ApplicationHistoryManager {
IOException;
/**
- * This method returns all Application {@link ApplicationReport}s
- *
+ * This method returns the given number of Application
+ * {@link ApplicationReport}s.
+ *
+ * @param appsNum
+ *
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
- Map<ApplicationId, ApplicationReport> getAllApplications()
- throws YarnException, IOException;
+ Map<ApplicationId, ApplicationReport>
+ getApplications(long appsNum) throws YarnException,
+ IOException;
/**
* Application can have multiple application attempts
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index c7cf07b..479858f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 5f4510b..3d8a19a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -78,6 +79,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
+ private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager,
@@ -91,6 +93,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ maxLoadedApplications =
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf);
}
@@ -101,12 +106,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
- ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
- null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
- UserGroupInformation.getLoginUser());
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+ null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+ EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index ba701a1..aba3af8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -57,6 +57,7 @@ public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
private final static int MAX_APPS = 2;
+ private static TimelineDataManager dataManager;
@BeforeClass
public static void setup() throws Exception {
@@ -64,7 +65,7 @@ public class TestApplicationHistoryClientService {
TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(MAX_APPS);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
- TimelineDataManager dataManager =
+ dataManager =
new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -169,8 +170,27 @@ public class TestApplicationHistoryClientService {
clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+ // Create a historyManager, and set the max_apps can be loaded
+ // as 1.
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ ApplicationHistoryManagerOnTimelineStore historyManager2 =
+ new ApplicationHistoryManagerOnTimelineStore(dataManager,
+ new ApplicationACLsManager(conf));
+ historyManager2.init(conf);
+ historyManager2.start();
+ @SuppressWarnings("resource")
+ ApplicationHistoryClientService clientService2 =
+ new ApplicationHistoryClientService(historyManager2);
+ response = clientService2.getApplications(request);
+ appReport = response.getApplicationList();
+ Assert.assertNotNull(appReport);
+ Assert.assertTrue(appReport.size() == 1);
+ // Expected to get the appReport for application with appId1
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index fcdafc2..8672953 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -180,8 +180,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
- Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
- Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L
+ + app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L
+ + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
@@ -335,7 +337,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
- historyManager.getAllApplications().values();
+ historyManager.getApplications(Long.MAX_VALUE).values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
ApplicationId ignoredAppId = ApplicationId.newInstance(0, SCALE + 2);
@@ -472,12 +474,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index a601975..3152165 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -77,9 +78,15 @@ public class AppsBlock extends HtmlBlock {
UserGroupInformation callerUGI = getCallerUGI();
Collection<ApplicationReport> appReports = null;
+
try {
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index d0ccd74..3064145 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -75,13 +75,11 @@ public class WebServices {
String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
- long num = 0;
- boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
- long countNum = 0;
+ long countNum = Long.MAX_VALUE;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
@@ -90,7 +88,6 @@ public class WebServices {
long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) {
- checkCount = true;
countNum = Long.parseLong(count);
if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0");
@@ -151,19 +148,20 @@ public class WebServices {
AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null;
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance();
+ request.setLimit(countNum);
try {
if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does
// in YARN-1819.
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
appReports = callerUGI.doAs(
new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
@Override
public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(
- GetApplicationsRequest.newInstance()).getApplicationList();
+ return appBaseProt.getApplications(request).getApplicationList();
}
});
}
@@ -172,10 +170,6 @@ public class WebServices {
}
for (ApplicationReport appReport : appReports) {
- if (checkCount && num == countNum) {
- break;
- }
-
if (checkAppStates &&
!appStates.contains(StringUtils.toLowerCase(
appReport.getYarnApplicationState().toString()))) {
@@ -215,7 +209,6 @@ public class WebServices {
AppInfo app = new AppInfo(appReport);
allApps.add(app);
- num++;
}
return allApps;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48bffd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index fde654b..edf1ff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -183,6 +183,7 @@ selected if this policy is either of `HTTPS_ONLY` or `HTTP_AND_HTTPS`.
| `yarn.timeline-service.handler-thread-count` | Handler thread count to serve the client RPC requests. Defaults to `10`. |
| `yarn.timeline-service.client.max-retries` | The maximum number of retries for attempts to publish data to the timeline service.Defaults to `30`. |
| `yarn.timeline-service.client.retry-interval-ms` | The interval in milliseconds between retries for the timeline service client. Defaults to `1000`. |
+| `yarn.timeline-service.generic-application-history.max-applications` | The max number of applications could be fetched by using REST API or application history protocol and shown in timeline server web ui. Defaults to `10000`. |