You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/09/03 22:25:11 UTC
[4/5] hadoop git commit: YARN-3700. Made generic history service load
a number of latest applications according to the parameter or the
configuration. Contributed by Xuan Gong.
YARN-3700. Made generic history service load a number of latest applications according to the parameter or the configuration. Contributed by Xuan Gong.
(cherry picked from commit 54504133f41e36eaea6bb06c7b9ddb249468ecd7)
(cherry picked from commit 839f81a6326b2f8b3d5183178382c1551b0bc259)
(cherry picked from commit 058380d9ef35f35e8c624fb8783eac0904c4d1f5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42ce0525
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42ce0525
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42ce0525
Branch: refs/heads/branch-2.6.1
Commit: 42ce052585dcb3ff5e954c41409c2a75d507bbb2
Parents: 0f33fcd
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed May 27 16:51:48 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 12:59:33 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 9 ++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../src/main/resources/yarn-default.xml | 9 ++++++
.../ApplicationHistoryClientService.java | 2 +-
.../ApplicationHistoryManager.java | 12 +++++---
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 13 +++++---
.../TestApplicationHistoryClientService.java | 26 ++++++++++++++--
...pplicationHistoryManagerOnTimelineStore.java | 12 ++++----
.../hadoop/yarn/server/webapp/AppsBlock.java | 6 ++++
.../hadoop/yarn/server/webapp/WebServices.java | 31 ++++++++------------
.../src/site/apt/TimelineServer.apt.vm | 8 +++++
13 files changed, 97 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e2419d6..328a0ed 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -144,6 +144,9 @@ Release 2.6.1 - UNRELEASED
YARN-2766. Made ApplicationHistoryManager return a sorted list of apps,
attempts and containers. (Robert Kanter via zjshen)
+ YARN-3700. Made generic history service load a number of latest applications
+ according to the parameter or the configuration. (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6d9c0da..bc89ed0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1285,6 +1285,15 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS =
"0.0.0.0:" + DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT;
+ /**
+ * Defines the max number of applications could be fetched using
+ * REST API or application history protocol and shown in timeline
+ * server web ui.
+ */
+ public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ APPLICATION_HISTORY_PREFIX + "max-applications";
+ public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
TIMELINE_SERVICE_PREFIX + "store-class";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 055f895..74b2415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -33,6 +33,7 @@ public interface YarnWebParams {
String ENTITY_STRING = "entity.string";
String APP_OWNER = "app.owner";
String APP_STATE = "app.state";
+ String APPS_NUM = "apps.num";
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String WEB_UI_TYPE = "web.ui.type";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 9d37196..e168600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1269,6 +1269,15 @@
</property>
<property>
+ <description>
+ Defines the max number of applications could be fetched using REST API or
+ application history protocol and shown in timeline server web ui.
+ </description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+ </property>
+
+ <property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
index 848224d..632853e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java
@@ -178,7 +178,7 @@ public class ApplicationHistoryClientService extends AbstractService implements
IOException {
GetApplicationsResponse response =
GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
- history.getAllApplications().values()));
+ history.getApplications(request.getLimit()).values()));
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
index 041c31b..c218eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManager.java
@@ -51,16 +51,20 @@ public interface ApplicationHistoryManager {
IOException;
/**
- * This method returns all Application {@link ApplicationReport}s
- *
+ * This method returns the given number of Application
+ * {@link ApplicationReport}s.
+ *
+ * @param appsNum
+ *
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
* @throws YarnException
* @throws IOException
*/
@Public
@Unstable
- Map<ApplicationId, ApplicationReport> getAllApplications()
- throws YarnException, IOException;
+ Map<ApplicationId, ApplicationReport>
+ getApplications(long appsNum) throws YarnException,
+ IOException;
/**
* Application can have multiple application attempts
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 803dc01..779b182 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -98,7 +98,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws IOException {
Map<ApplicationId, ApplicationHistoryData> histData =
historyStore.getAllApplications();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index f9ea14f..56ad469 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -73,6 +74,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
private TimelineDataManager timelineDataManager;
private ApplicationACLsManager aclsManager;
private String serverHttpAddress;
+ private long maxLoadedApplications;
public ApplicationHistoryManagerOnTimelineStore(
TimelineDataManager timelineDataManager,
@@ -86,6 +88,9 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
protected void serviceInit(Configuration conf) throws Exception {
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ maxLoadedApplications =
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
super.serviceInit(conf);
}
@@ -96,12 +101,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
}
@Override
- public Map<ApplicationId, ApplicationReport> getAllApplications()
+ public Map<ApplicationId, ApplicationReport> getApplications(long appsNum)
throws YarnException, IOException {
TimelineEntities entities = timelineDataManager.getEntities(
- ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
- null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
- UserGroupInformation.getLoginUser());
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null, null,
+ null, appsNum == Long.MAX_VALUE ? this.maxLoadedApplications : appsNum,
+ EnumSet.allOf(Field.class), UserGroupInformation.getLoginUser());
Map<ApplicationId, ApplicationReport> apps =
new LinkedHashMap<ApplicationId, ApplicationReport>();
if (entities != null && entities.getEntities() != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index b3fb3de..3a6214f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -53,6 +53,7 @@ import org.junit.Test;
public class TestApplicationHistoryClientService {
private static ApplicationHistoryClientService clientService;
+ private static TimelineDataManager dataManager;
@BeforeClass
public static void setup() throws Exception {
@@ -60,7 +61,7 @@ public class TestApplicationHistoryClientService {
TimelineStore store =
TestApplicationHistoryManagerOnTimelineStore.createStore(2);
TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
- TimelineDataManager dataManager =
+ dataManager =
new TimelineDataManager(store, aclsManager);
ApplicationACLsManager appAclsManager = new ApplicationACLsManager(conf);
ApplicationHistoryManagerOnTimelineStore historyManager =
@@ -97,8 +98,27 @@ public class TestApplicationHistoryClientService {
clientService.getApplications(request);
List<ApplicationReport> appReport = response.getApplicationList();
Assert.assertNotNull(appReport);
- Assert.assertEquals(appId, appReport.get(0).getApplicationId());
- Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId, appReport.get(1).getApplicationId());
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
+
+ // Create a historyManager, and set the max_apps can be loaded
+ // as 1.
+ Configuration conf = new YarnConfiguration();
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ ApplicationHistoryManagerOnTimelineStore historyManager2 =
+ new ApplicationHistoryManagerOnTimelineStore(dataManager,
+ new ApplicationACLsManager(conf));
+ historyManager2.init(conf);
+ historyManager2.start();
+ @SuppressWarnings("resource")
+ ApplicationHistoryClientService clientService2 =
+ new ApplicationHistoryClientService(historyManager2);
+ response = clientService2.getApplications(request);
+ appReport = response.getApplicationList();
+ Assert.assertNotNull(appReport);
+ Assert.assertTrue(appReport.size() == 1);
+ // Expected to get the appReport for application with appId1
+ Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ec96e0c..c44f22e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -175,8 +175,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test app type", app.getApplicationType());
Assert.assertEquals("user1", app.getUser());
Assert.assertEquals("test queue", app.getQueue());
- Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
- Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L
+ + app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L
+ + +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
// App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
// Nobody except admin and owner has access to the details of the app.
@@ -324,7 +326,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
@Test
public void testGetApplications() throws Exception {
Collection<ApplicationReport> apps =
- historyManager.getAllApplications().values();
+ historyManager.getApplications(Long.MAX_VALUE).values();
Assert.assertNotNull(apps);
Assert.assertEquals(SCALE + 1, apps.size());
}
@@ -450,12 +452,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L + appId.getId());
entity.addEvent(tEvent);
tEvent = new TimelineEvent();
tEvent.setEventType(
ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 3L + appId.getId());
Map<String, Object> eventInfo = new HashMap<String, Object>();
eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
"test diagnostics info");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 161486d..583a0ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APP_STATE;
+import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
@@ -80,6 +81,11 @@ public class AppsBlock extends HtmlBlock {
try {
final GetApplicationsRequest request =
GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
if (callerUGI == null) {
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
index a02b80f..0d6f166 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebServices.java
@@ -31,6 +31,13 @@ import javax.ws.rs.WebApplicationException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AuthorizationException;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -39,13 +46,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
-import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainersRequest;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptsInfo;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
@@ -71,13 +71,11 @@ public class WebServices {
String startedEnd, String finishBegin, String finishEnd,
Set<String> applicationTypes) {
UserGroupInformation callerUGI = getUser(req);
- long num = 0;
- boolean checkCount = false;
boolean checkStart = false;
boolean checkEnd = false;
boolean checkAppTypes = false;
boolean checkAppStates = false;
- long countNum = 0;
+ long countNum = Long.MAX_VALUE;
// set values suitable in case both of begin/end not specified
long sBegin = 0;
@@ -86,7 +84,6 @@ public class WebServices {
long fEnd = Long.MAX_VALUE;
if (count != null && !count.isEmpty()) {
- checkCount = true;
countNum = Long.parseLong(count);
if (countNum <= 0) {
throw new BadRequestException("limit value must be greater then 0");
@@ -147,19 +144,20 @@ public class WebServices {
AppsInfo allApps = new AppsInfo();
Collection<ApplicationReport> appReports = null;
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance();
+ request.setLimit(countNum);
try {
if (callerUGI == null) {
// TODO: the request should take the params like what RMWebServices does
// in YARN-1819.
- GetApplicationsRequest request = GetApplicationsRequest.newInstance();
appReports = appBaseProt.getApplications(request).getApplicationList();
} else {
appReports = callerUGI.doAs(
new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
@Override
public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(
- GetApplicationsRequest.newInstance()).getApplicationList();
+ return appBaseProt.getApplications(request).getApplicationList();
}
});
}
@@ -168,10 +166,6 @@ public class WebServices {
}
for (ApplicationReport appReport : appReports) {
- if (checkCount && num == countNum) {
- break;
- }
-
if (checkAppStates
&& !appStates.contains(appReport.getYarnApplicationState().toString()
.toLowerCase())) {
@@ -211,7 +205,6 @@ public class WebServices {
AppInfo app = new AppInfo(appReport);
allApps.add(app);
- num++;
}
return allApps;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ce0525/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
index 92c7377..7332b86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
@@ -104,6 +104,14 @@ YARN Timeline Server
</property>
<property>
+ <description>The max number of applications could be fetched by using REST API
+ or application history protocol and shown in timeline server web ui. Defaults
+ to `10000`.</description>
+ <name>yarn.timeline-service.generic-application-history.max-applications</name>
+ <value>10000</value>
+</property>
+
+<property>
<description>Enables cross-origin support (CORS) for web services where
cross-origin web response headers are needed. For example, javascript making
a web services request to the timeline server.</description>