You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2014/09/13 08:57:46 UTC
[06/12] git commit: YARN-2033. Merging generic-history into the
Timeline Store (Contributed by Zhijie Shen)
YARN-2033. Merging generic-history into the Timeline Store (Contributed by Zhijie Shen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b8b1608
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b8b1608
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b8b1608
Branch: refs/heads/HDFS-6584
Commit: 6b8b1608e64e300e4e1d23c60476febaca29ca38
Parents: 17ffbe0
Author: junping_du <ju...@apache.org>
Authored: Fri Sep 12 10:04:51 2014 +0800
Committer: junping_du <ju...@apache.org>
Committed: Fri Sep 12 10:04:51 2014 +0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/records/ApplicationAttemptReport.java | 18 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/client/ProtocolHATestBase.java | 2 +-
.../yarn/client/api/impl/TestAHSClient.java | 2 +
.../yarn/client/api/impl/TestYarnClient.java | 2 +
.../hadoop/yarn/client/cli/TestYarnCLI.java | 6 +-
.../impl/pb/ApplicationAttemptReportPBImpl.java | 19 +
.../hadoop/yarn/webapp/view/HtmlBlock.java | 12 +
.../src/main/resources/yarn-default.xml | 54 +-
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 511 +++++++++++++++++++
.../ApplicationHistoryServer.java | 11 +-
.../FileSystemApplicationHistoryStore.java | 3 +-
.../server/timeline/MemoryTimelineStore.java | 10 +-
.../TestApplicationHistoryClientService.java | 1 +
...pplicationHistoryManagerOnTimelineStore.java | 317 ++++++++++++
.../yarn/server/api/ApplicationContext.java | 43 +-
.../metrics/AppAttemptMetricsConstants.java | 64 +++
.../metrics/ApplicationMetricsConstants.java | 64 +++
.../metrics/ContainerMetricsConstants.java | 60 +++
.../yarn/server/webapp/AppAttemptBlock.java | 41 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 51 +-
.../hadoop/yarn/server/webapp/AppsBlock.java | 20 +-
.../yarn/server/webapp/ContainerBlock.java | 19 +-
.../hadoop/yarn/server/webapp/WebServices.java | 113 +++-
.../yarn/server/webapp/dao/AppAttemptInfo.java | 6 +
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 17 -
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 13 +
.../server/resourcemanager/ResourceManager.java | 9 +
.../ahs/RMApplicationHistoryWriter.java | 18 +-
.../metrics/AppAttemptFinishedEvent.java | 82 +++
.../metrics/AppAttemptRegisteredEvent.java | 81 +++
.../metrics/ApplicationCreatedEvent.java | 78 +++
.../metrics/ApplicationFinishedEvent.java | 75 +++
.../metrics/ContainerCreatedEvent.java | 67 +++
.../metrics/ContainerFinishedEvent.java | 65 +++
.../metrics/SystemMetricsEvent.java | 33 ++
.../metrics/SystemMetricsEventType.java | 34 ++
.../metrics/SystemMetricsPublisher.java | 490 ++++++++++++++++++
.../server/resourcemanager/rmapp/RMApp.java | 6 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 17 +
.../rmapp/attempt/RMAppAttemptImpl.java | 9 +-
.../rmcontainer/RMContainerImpl.java | 4 +
.../server/resourcemanager/TestAppManager.java | 3 +
.../resourcemanager/TestClientRMService.java | 3 +
.../applicationsmanager/MockAsm.java | 4 +
.../metrics/TestSystemMetricsPublisher.java | 355 +++++++++++++
.../server/resourcemanager/rmapp/MockRMApp.java | 10 +
.../rmapp/TestRMAppTransitions.java | 11 +
.../attempt/TestRMAppAttemptTransitions.java | 10 +
.../rmcontainer/TestRMContainerImpl.java | 11 +
.../scheduler/capacity/TestChildQueueOrder.java | 3 +
.../scheduler/capacity/TestUtils.java | 2 +
.../scheduler/fifo/TestFifoScheduler.java | 7 +
57 files changed, 2848 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5262a22..b8c6cf4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -73,6 +73,9 @@ Release 2.6.0 - UNRELEASED
YARN-2440. Enabled Nodemanagers to limit the aggregate cpu usage across all
containers to a preconfigured limit. (Varun Vasudev via vinodkv)
+ YARN-2033. YARN-2033. Merging generic-history into the Timeline Store
+ (Zhijie Shen via junping_du)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
index 031573f..53c18ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationAttemptReport.java
@@ -51,14 +51,15 @@ public abstract class ApplicationAttemptReport {
@Unstable
public static ApplicationAttemptReport newInstance(
ApplicationAttemptId applicationAttemptId, String host, int rpcPort,
- String url, String diagnostics, YarnApplicationAttemptState state,
- ContainerId amContainerId) {
+ String url, String oUrl, String diagnostics,
+ YarnApplicationAttemptState state, ContainerId amContainerId) {
ApplicationAttemptReport report =
Records.newRecord(ApplicationAttemptReport.class);
report.setApplicationAttemptId(applicationAttemptId);
report.setHost(host);
report.setRpcPort(rpcPort);
report.setTrackingUrl(url);
+ report.setOriginalTrackingUrl(oUrl);
report.setDiagnostics(diagnostics);
report.setYarnApplicationAttemptState(state);
report.setAMContainerId(amContainerId);
@@ -136,6 +137,19 @@ public abstract class ApplicationAttemptReport {
public abstract void setTrackingUrl(String url);
/**
+ * Get the <em>original tracking url</em> for the application attempt.
+ *
+ * @return <em>original tracking url</em> for the application attempt
+ */
+ @Public
+ @Unstable
+ public abstract String getOriginalTrackingUrl();
+
+ @Private
+ @Unstable
+ public abstract void setOriginalTrackingUrl(String oUrl);
+
+ /**
* Get the <code>ApplicationAttemptId</code> of this attempt of the
* application
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7c71a17..43f510d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -316,6 +316,19 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_RM_HISTORY_WRITER_MULTI_THREADED_DISPATCHER_POOL_SIZE =
10;
+ /**
+ * The setting that controls whether yarn system metrics is published on the
+ * timeline server or not by RM.
+ */
+ public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED =
+ RM_PREFIX + "system-metrics-publisher.enabled";
+ public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
+
+ public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
+ RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size";
+ public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
+ 10;
+
//Delegation token related keys
public static final String DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 7e7f21b..3b309ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -202,6 +202,7 @@ message ApplicationAttemptReportProto {
optional string diagnostics = 5 [default = "N/A"];
optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 6;
optional ContainerIdProto am_container_id = 7;
+ optional string original_tracking_url = 8;
}
enum NodeStateProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 72cb1b1..120538c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -676,7 +676,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes{
public ApplicationAttemptReport createFakeApplicationAttemptReport() {
return ApplicationAttemptReport.newInstance(
- createFakeApplicationAttemptId(), "localhost", 0, "", "",
+ createFakeApplicationAttemptId(), "localhost", 0, "", "", "",
YarnApplicationAttemptState.RUNNING, createFakeContainerId());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index 797faa5..d3c182b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -346,6 +346,7 @@ public class TestAHSClient {
"host",
124,
"url",
+ "oUrl",
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newInstance(
@@ -357,6 +358,7 @@ public class TestAHSClient {
"host",
124,
"url",
+ "oUrl",
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newInstance(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 6407f7a..8259893 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -457,6 +457,7 @@ public class TestYarnClient {
"host",
124,
"url",
+ "oUrl",
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newInstance(
@@ -467,6 +468,7 @@ public class TestYarnClient {
"host",
124,
"url",
+ "oUrl",
"diagnostics",
YarnApplicationAttemptState.FINISHED,
ContainerId.newInstance(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 47fa5ec..1a593d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -133,7 +133,7 @@ public class TestYarnCLI {
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
applicationId, 1);
ApplicationAttemptReport attemptReport = ApplicationAttemptReport
- .newInstance(attemptId, "host", 124, "url", "diagnostics",
+ .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics",
YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
attemptId, 1));
when(
@@ -169,11 +169,11 @@ public class TestYarnCLI {
ApplicationAttemptId attemptId1 = ApplicationAttemptId.newInstance(
applicationId, 2);
ApplicationAttemptReport attemptReport = ApplicationAttemptReport
- .newInstance(attemptId, "host", 124, "url", "diagnostics",
+ .newInstance(attemptId, "host", 124, "url", "oUrl", "diagnostics",
YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
attemptId, 1));
ApplicationAttemptReport attemptReport1 = ApplicationAttemptReport
- .newInstance(attemptId1, "host", 124, "url", "diagnostics",
+ .newInstance(attemptId1, "host", 124, "url", "oUrl", "diagnostics",
YarnApplicationAttemptState.FINISHED, ContainerId.newInstance(
attemptId1, 1));
List<ApplicationAttemptReport> reports = new ArrayList<ApplicationAttemptReport>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
index 8999987..c3c0221 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationAttemptReportPBImpl.java
@@ -88,6 +88,15 @@ public class ApplicationAttemptReportPBImpl extends ApplicationAttemptReport {
}
@Override
+ public String getOriginalTrackingUrl() {
+ ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasOriginalTrackingUrl()) {
+ return null;
+ }
+ return p.getOriginalTrackingUrl();
+ }
+
+ @Override
public String getDiagnostics() {
ApplicationAttemptReportProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasDiagnostics()) {
@@ -161,6 +170,16 @@ public class ApplicationAttemptReportPBImpl extends ApplicationAttemptReport {
}
@Override
+ public void setOriginalTrackingUrl(String oUrl) {
+ maybeInitBuilder();
+ if (oUrl == null) {
+ builder.clearOriginalTrackingUrl();
+ return;
+ }
+ builder.setOriginalTrackingUrl(oUrl);
+ }
+
+ @Override
public void setDiagnostics(String diagnostics) {
maybeInitBuilder();
if (diagnostics == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
index 8f885bb..6ee0d1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlBlock.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.webapp.view;
import java.io.PrintWriter;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.webapp.MimeType;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.WebAppException;
@@ -81,4 +82,15 @@ public abstract class HtmlBlock extends TextView implements SubView {
* @param html the block to render
*/
protected abstract void render(Block html);
+
+ protected UserGroupInformation getCallerUGI() {
+ // Check for the authorization.
+ String remoteUser = request().getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUGI;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 04e458c..55e5bca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -616,23 +616,31 @@
</property>
<property>
- <description>Number of worker threads that write the history data.</description>
- <name>yarn.resourcemanager.history-writer.multi-threaded-dispatcher.pool-size</name>
- <value>10</value>
- </property>
-
- <property>
<description>The class to use as the configuration provider.
If org.apache.hadoop.yarn.LocalConfigurationProvider is used,
the local configuration will be loaded.
If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used,
the configuration which will be loaded should be uploaded to remote File system first.
- </description>>
+ </description>
<name>yarn.resourcemanager.configuration.provider-class</name>
<value>org.apache.hadoop.yarn.LocalConfigurationProvider</value>
<!-- <value>org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider</value> -->
</property>
+ <property>
+ <description>The setting that controls whether yarn system metrics is
+ published on the timeline server or not by RM.</description>
+ <name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>Number of worker threads that send the yarn system metrics
+ data.</description>
+ <name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
+ <value>10</value>
+ </property>
+
<!-- Node Manager Configs -->
<property>
<description>The hostname of the NM.</description>
@@ -1319,38 +1327,6 @@
<value>/etc/krb5.keytab</value>
</property>
- <property>
- <description>Indicate to ResourceManager as well as clients whether
- history-service is enabled or not. If enabled, ResourceManager starts
- recording historical data that ApplicationHistory service can consume.
- Similarly, clients can redirect to the history service when applications
- finish if this is enabled.</description>
- <name>yarn.timeline-service.generic-application-history.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <description>URI pointing to the location of the FileSystem path where
- the history will be persisted. This must be supplied when using
- org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore
- as the value for yarn.timeline-service.generic-application-history.store-class</description>
- <name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name>
- <value>${hadoop.tmp.dir}/yarn/timeline/generic-history</value>
- </property>
-
- <property>
- <description>T-file compression types used to compress history data.</description>
- <name>yarn.timeline-service.generic-application-history.fs-history-store.compression-type</name>
- <value>none</value>
- </property>
-
- <property>
- <description>Store class name for history store, defaulting to file
- system store </description>
- <name>yarn.timeline-service.generic-application-history.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
- </property>
-
<!-- Other configuration -->
<property>
<description>The interval that the yarn client library uses to poll the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index b56a595..803dc01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -163,7 +163,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
ApplicationAttemptHistoryData appAttemptHistory) {
return ApplicationAttemptReport.newInstance(
appAttemptHistory.getApplicationAttemptId(), appAttemptHistory.getHost(),
- appAttemptHistory.getRPCPort(), appAttemptHistory.getTrackingURL(),
+ appAttemptHistory.getRPCPort(), appAttemptHistory.getTrackingURL(), null,
appAttemptHistory.getDiagnosticsInfo(),
appAttemptHistory.getYarnApplicationAttemptState(),
appAttemptHistory.getMasterContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
new file mode 100644
index 0000000..23b9218
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -0,0 +1,511 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.timeline.NameValuePair;
+import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
+import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
+ implements
+ ApplicationHistoryManager {
+
+ private TimelineDataManager timelineDataManager;
+ private String serverHttpAddress;
+
+ public ApplicationHistoryManagerOnTimelineStore(
+ TimelineDataManager timelineDataManager) {
+ super(ApplicationHistoryManagerOnTimelineStore.class.getName());
+ this.timelineDataManager = timelineDataManager;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
+ WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public ApplicationReport getApplication(ApplicationId appId)
+ throws YarnException, IOException {
+ return getApplication(appId, ApplicationReportField.ALL);
+ }
+
+ @Override
+ public Map<ApplicationId, ApplicationReport> getAllApplications()
+ throws YarnException, IOException {
+ TimelineEntities entities = timelineDataManager.getEntities(
+ ApplicationMetricsConstants.ENTITY_TYPE, null, null, null, null,
+ null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ Map<ApplicationId, ApplicationReport> apps =
+ new HashMap<ApplicationId, ApplicationReport>();
+ if (entities != null && entities.getEntities() != null) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ ApplicationReport app =
+ generateApplicationReport(entity, ApplicationReportField.ALL);
+ apps.put(app.getApplicationId(), app);
+ }
+ }
+ return apps;
+ }
+
+ @Override
+ public Map<ApplicationAttemptId, ApplicationAttemptReport>
+ getApplicationAttempts(ApplicationId appId)
+ throws YarnException, IOException {
+ TimelineEntities entities = timelineDataManager.getEntities(
+ AppAttemptMetricsConstants.ENTITY_TYPE,
+ new NameValuePair(
+ AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER, appId
+ .toString()), null, null, null, null, null,
+ Long.MAX_VALUE, EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ Map<ApplicationAttemptId, ApplicationAttemptReport> appAttempts =
+ new HashMap<ApplicationAttemptId, ApplicationAttemptReport>();
+ if (entities != null && entities.getEntities() != null) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ ApplicationAttemptReport appAttempt =
+ convertToApplicationAttemptReport(entity);
+ appAttempts.put(appAttempt.getApplicationAttemptId(), appAttempt);
+ }
+ } else {
+ // It is likely that the attemtps are not found due to non-existing
+ // application. In this case, we need to throw ApplicationNotFoundException.
+ getApplication(appId, ApplicationReportField.NONE);
+ }
+ return appAttempts;
+ }
+
+ @Override
+ public ApplicationAttemptReport getApplicationAttempt(
+ ApplicationAttemptId appAttemptId) throws YarnException, IOException {
+ TimelineEntity entity = timelineDataManager.getEntity(
+ AppAttemptMetricsConstants.ENTITY_TYPE,
+ appAttemptId.toString(), EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ if (entity == null) {
+ // Will throw ApplicationNotFoundException first
+ getApplication(appAttemptId.getApplicationId(), ApplicationReportField.NONE);
+ throw new ApplicationAttemptNotFoundException(
+ "The entity for application attempt " + appAttemptId +
+ " doesn't exist in the timeline store");
+ } else {
+ return convertToApplicationAttemptReport(entity);
+ }
+ }
+
+ @Override
+ public ContainerReport getContainer(ContainerId containerId)
+ throws YarnException, IOException {
+ ApplicationReport app = getApplication(
+ containerId.getApplicationAttemptId().getApplicationId(),
+ ApplicationReportField.USER);
+ TimelineEntity entity = timelineDataManager.getEntity(
+ ContainerMetricsConstants.ENTITY_TYPE,
+ containerId.toString(), EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ if (entity == null) {
+ throw new ContainerNotFoundException(
+ "The entity for container " + containerId +
+ " doesn't exist in the timeline store");
+ } else {
+ return convertToContainerReport(entity, serverHttpAddress, app.getUser());
+ }
+ }
+
+ @Override
+ public ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
+ throws YarnException, IOException {
+ ApplicationAttemptReport appAttempt = getApplicationAttempt(appAttemptId);
+ return getContainer(appAttempt.getAMContainerId());
+ }
+
+ @Override
+ public Map<ContainerId, ContainerReport> getContainers(
+ ApplicationAttemptId appAttemptId) throws YarnException, IOException {
+ ApplicationReport app = getApplication(
+ appAttemptId.getApplicationId(), ApplicationReportField.USER);
+ TimelineEntities entities = timelineDataManager.getEntities(
+ ContainerMetricsConstants.ENTITY_TYPE,
+ new NameValuePair(
+ ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
+ appAttemptId.toString()), null, null, null,
+ null, null, Long.MAX_VALUE, EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ Map<ContainerId, ContainerReport> containers =
+ new HashMap<ContainerId, ContainerReport>();
+ if (entities != null && entities.getEntities() != null) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ ContainerReport container =
+ convertToContainerReport(entity, serverHttpAddress, app.getUser());
+ containers.put(container.getContainerId(), container);
+ }
+ }
+ return containers;
+ }
+
+ private static ApplicationReport convertToApplicationReport(
+ TimelineEntity entity, ApplicationReportField field) {
+ String user = null;
+ String queue = null;
+ String name = null;
+ String type = null;
+ long createdTime = 0;
+ long finishedTime = 0;
+ ApplicationAttemptId latestApplicationAttemptId = null;
+ String diagnosticsInfo = null;
+ FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
+ YarnApplicationState state = null;
+ if (field == ApplicationReportField.NONE) {
+ return ApplicationReport.newInstance(
+ ConverterUtils.toApplicationId(entity.getEntityId()),
+ latestApplicationAttemptId, user, queue, name, null, -1, null, state,
+ diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
+ null, 1.0F, type, null);
+ }
+ Map<String, Object> entityInfo = entity.getOtherInfo();
+ if (entityInfo != null) {
+ if (entityInfo.containsKey(ApplicationMetricsConstants.USER_ENTITY_INFO)) {
+ user =
+ entityInfo.get(ApplicationMetricsConstants.USER_ENTITY_INFO)
+ .toString();
+ }
+ if (field == ApplicationReportField.USER) {
+ return ApplicationReport.newInstance(
+ ConverterUtils.toApplicationId(entity.getEntityId()),
+ latestApplicationAttemptId, user, queue, name, null, -1, null, state,
+ diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
+ null, 1.0F, type, null);
+ }
+ if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
+ queue =
+ entityInfo.get(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)
+ .toString();
+ }
+ if (entityInfo.containsKey(ApplicationMetricsConstants.NAME_ENTITY_INFO)) {
+ name =
+ entityInfo.get(ApplicationMetricsConstants.NAME_ENTITY_INFO)
+ .toString();
+ }
+ if (entityInfo.containsKey(ApplicationMetricsConstants.TYPE_ENTITY_INFO)) {
+ type =
+ entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
+ .toString();
+ }
+ }
+ List<TimelineEvent> events = entity.getEvents();
+ if (events != null) {
+ for (TimelineEvent event : events) {
+ if (event.getEventType().equals(
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
+ createdTime = event.getTimestamp();
+ } else if (event.getEventType().equals(
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ finishedTime = event.getTimestamp();
+ Map<String, Object> eventInfo = event.getEventInfo();
+ if (eventInfo == null) {
+ continue;
+ }
+ if (eventInfo
+ .containsKey(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)) {
+ latestApplicationAttemptId =
+ ConverterUtils
+ .toApplicationAttemptId(
+ eventInfo
+ .get(
+ ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO)
+ .toString());
+ }
+ if (eventInfo
+ .containsKey(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
+ diagnosticsInfo =
+ eventInfo.get(
+ ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)
+ .toString();
+ }
+ if (eventInfo
+ .containsKey(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)) {
+ finalStatus =
+ FinalApplicationStatus.valueOf(eventInfo.get(
+ ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO)
+ .toString());
+ }
+ if (eventInfo
+ .containsKey(ApplicationMetricsConstants.STATE_EVENT_INFO)) {
+ state =
+ YarnApplicationState.valueOf(eventInfo.get(
+ ApplicationMetricsConstants.STATE_EVENT_INFO).toString());
+ }
+ }
+ }
+ }
+ return ApplicationReport.newInstance(
+ ConverterUtils.toApplicationId(entity.getEntityId()),
+ latestApplicationAttemptId, user, queue, name, null, -1, null, state,
+ diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
+ null, 1.0F, type, null);
+ }
+
+ private static ApplicationAttemptReport convertToApplicationAttemptReport(
+ TimelineEntity entity) {
+ String host = null;
+ int rpcPort = -1;
+ ContainerId amContainerId = null;
+ String trackingUrl = null;
+ String originalTrackingUrl = null;
+ String diagnosticsInfo = null;
+ YarnApplicationAttemptState state = null;
+ List<TimelineEvent> events = entity.getEvents();
+ if (events != null) {
+ for (TimelineEvent event : events) {
+ if (event.getEventType().equals(
+ AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE)) {
+ Map<String, Object> eventInfo = event.getEventInfo();
+ if (eventInfo == null) {
+ continue;
+ }
+ if (eventInfo.containsKey(AppAttemptMetricsConstants.HOST_EVENT_INFO)) {
+ host =
+ eventInfo.get(AppAttemptMetricsConstants.HOST_EVENT_INFO)
+ .toString();
+ }
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO)) {
+ rpcPort = (Integer) eventInfo.get(
+ AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO);
+ }
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
+ amContainerId =
+ ConverterUtils.toContainerId(eventInfo.get(
+ AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
+ .toString());
+ }
+ } else if (event.getEventType().equals(
+ AppAttemptMetricsConstants.FINISHED_EVENT_TYPE)) {
+ Map<String, Object> eventInfo = event.getEventInfo();
+ if (eventInfo == null) {
+ continue;
+ }
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO)) {
+ trackingUrl =
+ eventInfo.get(
+ AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO)
+ .toString();
+ }
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO)) {
+ originalTrackingUrl =
+ eventInfo
+ .get(
+ AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO)
+ .toString();
+ }
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
+ diagnosticsInfo =
+ eventInfo.get(
+ AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)
+ .toString();
+ }
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.STATE_EVENT_INFO)) {
+ state =
+ YarnApplicationAttemptState.valueOf(eventInfo.get(
+ AppAttemptMetricsConstants.STATE_EVENT_INFO)
+ .toString());
+ }
+ }
+ }
+ }
+ return ApplicationAttemptReport.newInstance(
+ ConverterUtils.toApplicationAttemptId(entity.getEntityId()),
+ host, rpcPort, trackingUrl, originalTrackingUrl, diagnosticsInfo,
+ state, amContainerId);
+ }
+
+ private static ContainerReport convertToContainerReport(
+ TimelineEntity entity, String serverHttpAddress, String user) {
+ int allocatedMem = 0;
+ int allocatedVcore = 0;
+ String allocatedHost = null;
+ int allocatedPort = -1;
+ int allocatedPriority = 0;
+ long createdTime = 0;
+ long finishedTime = 0;
+ String diagnosticsInfo = null;
+ int exitStatus = ContainerExitStatus.INVALID;
+ ContainerState state = null;
+ Map<String, Object> entityInfo = entity.getOtherInfo();
+ if (entityInfo != null) {
+ if (entityInfo
+ .containsKey(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO)) {
+ allocatedMem = (Integer) entityInfo.get(
+ ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO);
+ }
+ if (entityInfo
+ .containsKey(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO)) {
+ allocatedVcore = (Integer) entityInfo.get(
+ ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO);
+ }
+ if (entityInfo
+ .containsKey(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)) {
+ allocatedHost =
+ entityInfo
+ .get(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO)
+ .toString();
+ }
+ if (entityInfo
+ .containsKey(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO)) {
+ allocatedPort = (Integer) entityInfo.get(
+ ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO);
+ }
+ if (entityInfo
+ .containsKey(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO)) {
+ allocatedPriority = (Integer) entityInfo.get(
+ ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO);
+ }
+ }
+ List<TimelineEvent> events = entity.getEvents();
+ if (events != null) {
+ for (TimelineEvent event : events) {
+ if (event.getEventType().equals(
+ ContainerMetricsConstants.CREATED_EVENT_TYPE)) {
+ createdTime = event.getTimestamp();
+ } else if (event.getEventType().equals(
+ ContainerMetricsConstants.FINISHED_EVENT_TYPE)) {
+ finishedTime = event.getTimestamp();
+ Map<String, Object> eventInfo = event.getEventInfo();
+ if (eventInfo == null) {
+ continue;
+ }
+ if (eventInfo
+ .containsKey(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)) {
+ diagnosticsInfo =
+ eventInfo.get(
+ ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO)
+ .toString();
+ }
+ if (eventInfo
+ .containsKey(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO)) {
+ exitStatus = (Integer) eventInfo.get(
+ ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO);
+ }
+ if (eventInfo
+ .containsKey(ContainerMetricsConstants.STATE_EVENT_INFO)) {
+ state =
+ ContainerState.valueOf(eventInfo.get(
+ ContainerMetricsConstants.STATE_EVENT_INFO).toString());
+ }
+ }
+ }
+ }
+ NodeId allocatedNode = NodeId.newInstance(allocatedHost, allocatedPort);
+ ContainerId containerId =
+ ConverterUtils.toContainerId(entity.getEntityId());
+ String logUrl = WebAppUtils.getAggregatedLogURL(
+ serverHttpAddress,
+ allocatedNode.toString(),
+ containerId.toString(),
+ containerId.toString(),
+ user);
+ return ContainerReport.newInstance(
+ ConverterUtils.toContainerId(entity.getEntityId()),
+ Resource.newInstance(allocatedMem, allocatedVcore),
+ NodeId.newInstance(allocatedHost, allocatedPort),
+ Priority.newInstance(allocatedPriority),
+ createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state);
+ }
+
+ private ApplicationReport generateApplicationReport(TimelineEntity entity,
+ ApplicationReportField field) throws YarnException, IOException {
+ ApplicationReport app = convertToApplicationReport(entity, field);
+ if (field == ApplicationReportField.ALL &&
+ app != null && app.getCurrentApplicationAttemptId() != null) {
+ ApplicationAttemptReport appAttempt =
+ getApplicationAttempt(app.getCurrentApplicationAttemptId());
+ if (appAttempt != null) {
+ app.setHost(appAttempt.getHost());
+ app.setRpcPort(appAttempt.getRpcPort());
+ app.setTrackingUrl(appAttempt.getTrackingUrl());
+ app.setOriginalTrackingUrl(appAttempt.getOriginalTrackingUrl());
+ }
+ }
+ return app;
+ }
+
+ private ApplicationReport getApplication(ApplicationId appId,
+ ApplicationReportField field) throws YarnException, IOException {
+ TimelineEntity entity = timelineDataManager.getEntity(
+ ApplicationMetricsConstants.ENTITY_TYPE,
+ appId.toString(), EnumSet.allOf(Field.class),
+ UserGroupInformation.getLoginUser());
+ if (entity == null) {
+ throw new ApplicationNotFoundException("The entity for application " +
+ appId + " doesn't exist in the timeline store");
+ } else {
+ return generateApplicationReport(entity, field);
+ }
+ }
+
+ private static enum ApplicationReportField {
+ ALL, // retrieve all the fields
+ NONE, // retrieve no fields
+ USER // retrieve user info only
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index 158f2e6..3c3d37a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -170,7 +170,16 @@ public class ApplicationHistoryServer extends CompositeService {
private ApplicationHistoryManager createApplicationHistoryManager(
Configuration conf) {
- return new ApplicationHistoryManagerImpl();
+ // Backward compatibility:
+ // APPLICATION_HISTORY_STORE is neither null nor empty, it means that the
+ // user has enabled it explicitly.
+ if (conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE) == null ||
+ conf.get(YarnConfiguration.APPLICATION_HISTORY_STORE).length() == 0) {
+ return new ApplicationHistoryManagerOnTimelineStore(timelineDataManager);
+ } else {
+ LOG.warn("The filesystem based application history store is deprecated.");
+ return new ApplicationHistoryManagerImpl();
+ }
}
private TimelineStore createTimelineStore(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
index a2d9140..7840dd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
@@ -117,7 +117,8 @@ public class FileSystemApplicationHistoryStore extends AbstractService
@Override
public void serviceInit(Configuration conf) throws Exception {
Path fsWorkingPath =
- new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI));
+ new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
+ conf.get("hadoop.tmp.dir") + "/yarn/timeline/generic-history"));
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
try {
fs = getFileSystem(fsWorkingPath, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
index b94711c..099d5ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/MemoryTimelineStore.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelineP
* they may encounter reading and writing history data in different memory
* store.
*
+ * The methods are synchronized to avoid concurrent modification on the memory.
+ *
*/
@Private
@Unstable
@@ -65,7 +67,7 @@ public class MemoryTimelineStore
}
@Override
- public TimelineEntities getEntities(String entityType, Long limit,
+ public synchronized TimelineEntities getEntities(String entityType, Long limit,
Long windowStart, Long windowEnd, String fromId, Long fromTs,
NameValuePair primaryFilter, Collection<NameValuePair> secondaryFilters,
EnumSet<Field> fields) {
@@ -148,7 +150,7 @@ public class MemoryTimelineStore
}
@Override
- public TimelineEntity getEntity(String entityId, String entityType,
+ public synchronized TimelineEntity getEntity(String entityId, String entityType,
EnumSet<Field> fieldsToRetrieve) {
if (fieldsToRetrieve == null) {
fieldsToRetrieve = EnumSet.allOf(Field.class);
@@ -162,7 +164,7 @@ public class MemoryTimelineStore
}
@Override
- public TimelineEvents getEntityTimelines(String entityType,
+ public synchronized TimelineEvents getEntityTimelines(String entityType,
SortedSet<String> entityIds, Long limit, Long windowStart,
Long windowEnd,
Set<String> eventTypes) {
@@ -209,7 +211,7 @@ public class MemoryTimelineStore
}
@Override
- public TimelinePutResponse put(TimelineEntities data) {
+ public synchronized TimelinePutResponse put(TimelineEntities data) {
TimelinePutResponse response = new TimelinePutResponse();
for (TimelineEntity entity : data.getEntities()) {
EntityIdentifier entityId =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7b5b74b..ad2907b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -64,6 +64,7 @@ public class TestApplicationHistoryClientService extends
WebAppUtils.getAHSWebAppURLWithoutScheme(config) +
"/applicationhistory/logs/localhost:0/container_0_0001_01_000001/" +
"container_0_0001_01_000001/test user";
+ config.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED, true);
config.setClass(YarnConfiguration.APPLICATION_HISTORY_STORE,
MemoryApplicationHistoryStore.class, ApplicationHistoryStore.class);
historyServer.init(config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
new file mode 100644
index 0000000..65eafc6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -0,0 +1,317 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
+import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
+import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestApplicationHistoryManagerOnTimelineStore {
+
+ private static ApplicationHistoryManagerOnTimelineStore historyManager;
+ private static final int SCALE = 5;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ TimelineStore store = new MemoryTimelineStore();
+ prepareTimelineStore(store);
+ TimelineACLsManager aclsManager = new TimelineACLsManager(conf);
+ TimelineDataManager dataManager =
+ new TimelineDataManager(store, aclsManager);
+ historyManager = new ApplicationHistoryManagerOnTimelineStore(dataManager);
+ historyManager.init(conf);
+ historyManager.start();
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ if (historyManager != null) {
+ historyManager.stop();
+ }
+ }
+
+ private static void prepareTimelineStore(TimelineStore store)
+ throws Exception {
+ for (int i = 1; i <= SCALE; ++i) {
+ TimelineEntities entities = new TimelineEntities();
+ ApplicationId appId = ApplicationId.newInstance(0, i);
+ entities.addEntity(createApplicationTimelineEntity(appId));
+ store.put(entities);
+ for (int j = 1; j <= SCALE; ++j) {
+ entities = new TimelineEntities();
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, j);
+ entities.addEntity(createAppAttemptTimelineEntity(appAttemptId));
+ store.put(entities);
+ for (int k = 1; k <= SCALE; ++k) {
+ entities = new TimelineEntities();
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, k);
+ entities.addEntity(createContainerEntity(containerId));
+ store.put(entities);
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testGetApplicationReport() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationReport app = historyManager.getApplication(appId);
+ Assert.assertNotNull(app);
+ Assert.assertEquals(appId, app.getApplicationId());
+ Assert.assertEquals("test app", app.getName());
+ Assert.assertEquals("test app type", app.getApplicationType());
+ Assert.assertEquals("test user", app.getUser());
+ Assert.assertEquals("test queue", app.getQueue());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
+ Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+ Assert.assertEquals("test host", app.getHost());
+ Assert.assertEquals(-100, app.getRpcPort());
+ Assert.assertEquals("test tracking url", app.getTrackingUrl());
+ Assert.assertEquals("test original tracking url",
+ app.getOriginalTrackingUrl());
+ Assert.assertEquals("test diagnostics info", app.getDiagnostics());
+ Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
+ app.getFinalApplicationStatus());
+ Assert.assertEquals(YarnApplicationState.FINISHED,
+ app.getYarnApplicationState());
+ }
+
+ @Test
+ public void testGetApplicationAttemptReport() throws Exception {
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
+ ApplicationAttemptReport appAttempt =
+ historyManager.getApplicationAttempt(appAttemptId);
+ Assert.assertNotNull(appAttempt);
+ Assert.assertEquals(appAttemptId, appAttempt.getApplicationAttemptId());
+ Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
+ appAttempt.getAMContainerId());
+ Assert.assertEquals("test host", appAttempt.getHost());
+ Assert.assertEquals(-100, appAttempt.getRpcPort());
+ Assert.assertEquals("test tracking url", appAttempt.getTrackingUrl());
+ Assert.assertEquals("test original tracking url",
+ appAttempt.getOriginalTrackingUrl());
+ Assert.assertEquals("test diagnostics info", appAttempt.getDiagnostics());
+ Assert.assertEquals(YarnApplicationAttemptState.FINISHED,
+ appAttempt.getYarnApplicationAttemptState());
+ }
+
+ @Test
+ public void testGetContainerReport() throws Exception {
+ ContainerId containerId =
+ ContainerId.newInstance(ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0, 1), 1), 1);
+ ContainerReport container = historyManager.getContainer(containerId);
+ Assert.assertNotNull(container);
+ Assert.assertEquals(Integer.MAX_VALUE + 1L, container.getCreationTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 2L, container.getFinishTime());
+ Assert.assertEquals(Resource.newInstance(-1, -1),
+ container.getAllocatedResource());
+ Assert.assertEquals(NodeId.newInstance("test host", -100),
+ container.getAssignedNode());
+ Assert.assertEquals(Priority.UNDEFINED, container.getPriority());
+ Assert
+ .assertEquals("test diagnostics info", container.getDiagnosticsInfo());
+ Assert.assertEquals(ContainerState.COMPLETE, container.getContainerState());
+ Assert.assertEquals(-1, container.getContainerExitStatus());
+ Assert.assertEquals("http://0.0.0.0:8188/applicationhistory/logs/" +
+ "test host:-100/container_0_0001_01_000001/"
+ + "container_0_0001_01_000001/test user", container.getLogUrl());
+ }
+
+ @Test
+ public void testGetApplications() throws Exception {
+ Collection<ApplicationReport> apps =
+ historyManager.getAllApplications().values();
+ Assert.assertNotNull(apps);
+ Assert.assertEquals(SCALE, apps.size());
+ }
+
+ @Test
+ public void testGetApplicationAttempts() throws Exception {
+ Collection<ApplicationAttemptReport> appAttempts =
+ historyManager.getApplicationAttempts(ApplicationId.newInstance(0, 1))
+ .values();
+ Assert.assertNotNull(appAttempts);
+ Assert.assertEquals(SCALE, appAttempts.size());
+ }
+
+ @Test
+ public void testGetContainers() throws Exception {
+ Collection<ContainerReport> containers =
+ historyManager
+ .getContainers(
+ ApplicationAttemptId.newInstance(
+ ApplicationId.newInstance(0, 1), 1)).values();
+ Assert.assertNotNull(containers);
+ Assert.assertEquals(SCALE, containers.size());
+ }
+
+ @Test
+ public void testGetAMContainer() throws Exception {
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
+ ContainerReport container = historyManager.getAMContainer(appAttemptId);
+ Assert.assertNotNull(container);
+ Assert.assertEquals(appAttemptId, container.getContainerId()
+ .getApplicationAttemptId());
+ }
+
+ private static TimelineEntity createApplicationTimelineEntity(
+ ApplicationId appId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(appId.toString());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO, "test app");
+ entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
+ "test app type");
+ entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO, "test user");
+ entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO, "test queue");
+ entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
+ Integer.MAX_VALUE + 1L);
+ entity.setOtherInfo(entityInfo);
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ entity.addEvent(tEvent);
+ tEvent = new TimelineEvent();
+ tEvent.setEventType(
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 3L);
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ "test diagnostics info");
+ eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
+ FinalApplicationStatus.UNDEFINED.toString());
+ eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
+ YarnApplicationState.FINISHED.toString());
+ eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
+ ApplicationAttemptId.newInstance(appId, 1));
+ tEvent.setEventInfo(eventInfo);
+ entity.addEvent(tEvent);
+ return entity;
+ }
+
+ private static TimelineEntity createAppAttemptTimelineEntity(
+ ApplicationAttemptId appAttemptId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(appAttemptId.toString());
+ entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
+ appAttemptId.getApplicationId().toString());
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ "test tracking url");
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ "test original tracking url");
+ eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, "test host");
+ eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO, -100);
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ ContainerId.newInstance(appAttemptId, 1));
+ tEvent.setEventInfo(eventInfo);
+ entity.addEvent(tEvent);
+ tEvent = new TimelineEvent();
+ tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ "test tracking url");
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ "test original tracking url");
+ eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ "test diagnostics info");
+ eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO,
+ FinalApplicationStatus.UNDEFINED.toString());
+ eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
+ YarnApplicationAttemptState.FINISHED.toString());
+ tEvent.setEventInfo(eventInfo);
+ entity.addEvent(tEvent);
+ return entity;
+ }
+
+ private static TimelineEntity createContainerEntity(ContainerId containerId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(ContainerMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(containerId.toString());
+ entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
+ containerId.getApplicationAttemptId().toString());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, -1);
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, -1);
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
+ "test host");
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, -100);
+ entityInfo
+ .put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO, -1);
+ entity.setOtherInfo(entityInfo);
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 1L);
+ entity.addEvent(tEvent);
+ ;
+ tEvent = new TimelineEvent();
+ tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(Integer.MAX_VALUE + 2L);
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ "test diagnostics info");
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO, -1);
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
+ ContainerState.COMPLETE.toString());
+ tEvent.setEventInfo(eventInfo);
+ entity.addEvent(tEvent);
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java
index 78ae0dd..0e2ffdf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/ApplicationContext.java
@@ -21,17 +21,14 @@ package org.apache.hadoop.yarn.server.api;
import java.io.IOException;
import java.util.Map;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.exceptions.YarnException;
-@Public
-@Unstable
public interface ApplicationContext {
/**
* This method returns Application {@link ApplicationReport} for the specified
@@ -40,21 +37,21 @@ public interface ApplicationContext {
* @param appId
*
* @return {@link ApplicationReport} for the ApplicationId.
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
- ApplicationReport getApplication(ApplicationId appId) throws IOException;
+ ApplicationReport getApplication(ApplicationId appId)
+ throws YarnException, IOException;
/**
* This method returns all Application {@link ApplicationReport}s
*
* @return map of {@link ApplicationId} to {@link ApplicationReport}s.
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
- Map<ApplicationId, ApplicationReport> getAllApplications() throws IOException;
+ Map<ApplicationId, ApplicationReport> getAllApplications()
+ throws YarnException, IOException;
/**
* Application can have multiple application attempts
@@ -64,12 +61,11 @@ public interface ApplicationContext {
* @param appId
*
* @return all {@link ApplicationAttemptReport}s for the Application.
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
Map<ApplicationAttemptId, ApplicationAttemptReport> getApplicationAttempts(
- ApplicationId appId) throws IOException;
+ ApplicationId appId) throws YarnException, IOException;
/**
* This method returns {@link ApplicationAttemptReport} for specified
@@ -78,12 +74,11 @@ public interface ApplicationContext {
* @param appAttemptId
* {@link ApplicationAttemptId}
* @return {@link ApplicationAttemptReport} for ApplicationAttemptId
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
ApplicationAttemptReport getApplicationAttempt(
- ApplicationAttemptId appAttemptId) throws IOException;
+ ApplicationAttemptId appAttemptId) throws YarnException, IOException;
/**
* This method returns {@link ContainerReport} for specified
@@ -92,11 +87,11 @@ public interface ApplicationContext {
* @param containerId
* {@link ContainerId}
* @return {@link ContainerReport} for ContainerId
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
- ContainerReport getContainer(ContainerId containerId) throws IOException;
+ ContainerReport getContainer(ContainerId containerId)
+ throws YarnException, IOException;
/**
* This method returns {@link ContainerReport} for specified
@@ -105,12 +100,11 @@ public interface ApplicationContext {
* @param appAttemptId
* {@link ApplicationAttemptId}
* @return {@link ContainerReport} for ApplicationAttemptId
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
ContainerReport getAMContainer(ApplicationAttemptId appAttemptId)
- throws IOException;
+ throws YarnException, IOException;
/**
* This method returns Map of {@link ContainerId} to {@link ContainerReport}
@@ -120,10 +114,9 @@ public interface ApplicationContext {
* {@link ApplicationAttemptId}
* @return Map of {@link ContainerId} to {@link ContainerReport} for
* ApplicationAttemptId
+ * @throws YarnException
* @throws IOException
*/
- @Public
- @Unstable
Map<ContainerId, ContainerReport> getContainers(
- ApplicationAttemptId appAttemptId) throws IOException;
+ ApplicationAttemptId appAttemptId) throws YarnException, IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java
new file mode 100644
index 0000000..a7809cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/AppAttemptMetricsConstants.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
+public class AppAttemptMetricsConstants {
+
+ public static final String ENTITY_TYPE =
+ "YARN_APPLICATION_ATTEMPT";
+
+ public static final String REGISTERED_EVENT_TYPE =
+ "YARN_APPLICATION_ATTEMPT_REGISTERED";
+
+ public static final String FINISHED_EVENT_TYPE =
+ "YARN_APPLICATION_ATTEMPT_FINISHED";
+
+ public static final String PARENT_PRIMARY_FILTER =
+ "YARN_APPLICATION_ATTEMPT_PARENT";
+
+ public static final String TRACKING_URL_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_TRACKING_URL";
+
+ public static final String ORIGINAL_TRACKING_URL_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_ORIGINAL_TRACKING_URL";
+
+ public static final String HOST_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_HOST";
+
+ public static final String RPC_PORT_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_RPC_PORT";
+
+ public static final String MASTER_CONTAINER_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_MASTER_CONTAINER";
+
+ public static final String DIAGNOSTICS_INFO_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_DIAGNOSTICS_INFO";
+
+ public static final String FINAL_STATUS_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_FINAL_STATUS";
+
+ public static final String STATE_EVENT_INFO =
+ "YARN_APPLICATION_ATTEMPT_STATE";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
new file mode 100644
index 0000000..f6a40bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
+public class ApplicationMetricsConstants {
+
+ public static final String ENTITY_TYPE =
+ "YARN_APPLICATION";
+
+ public static final String CREATED_EVENT_TYPE =
+ "YARN_APPLICATION_CREATED";
+
+ public static final String FINISHED_EVENT_TYPE =
+ "YARN_APPLICATION_FINISHED";
+
+ public static final String NAME_ENTITY_INFO =
+ "YARN_APPLICATION_NAME";
+
+ public static final String TYPE_ENTITY_INFO =
+ "YARN_APPLICATION_TYPE";
+
+ public static final String USER_ENTITY_INFO =
+ "YARN_APPLICATION_USER";
+
+ public static final String QUEUE_ENTITY_INFO =
+ "YARN_APPLICATION_QUEUE";
+
+ public static final String SUBMITTED_TIME_ENTITY_INFO =
+ "YARN_APPLICATION_SUBMITTED_TIME";
+
+ public static final String DIAGNOSTICS_INFO_EVENT_INFO =
+ "YARN_APPLICATION_DIAGNOSTICS_INFO";
+
+ public static final String FINAL_STATUS_EVENT_INFO =
+ "YARN_APPLICATION_FINAL_STATUS";
+
+ public static final String STATE_EVENT_INFO =
+ "YARN_APPLICATION_STATE";
+
+ public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
+ "YARN_APPLICATION_LATEST_APP_ATTEMPT";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b8b1608/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
new file mode 100644
index 0000000..8791da4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+@Private
+@Unstable
+public class ContainerMetricsConstants {
+
+ public static final String ENTITY_TYPE = "YARN_CONTAINER";
+
+ public static final String CREATED_EVENT_TYPE = "YARN_CONTAINER_CREATED";
+
+ public static final String FINISHED_EVENT_TYPE = "YARN_CONTAINER_FINISHED";
+
+ public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT";
+
+ public static final String ALLOCATED_MEMORY_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_MEMORY";
+
+ public static final String ALLOCATED_VCORE_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_VCORE";
+
+ public static final String ALLOCATED_HOST_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_HOST";
+
+ public static final String ALLOCATED_PORT_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_PORT";
+
+ public static final String ALLOCATED_PRIORITY_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_PRIORITY";
+
+ public static final String DIAGNOSTICS_INFO_EVENT_INFO =
+ "YARN_CONTAINER_DIAGNOSTICS_INFO";
+
+ public static final String EXIT_STATUS_EVENT_INFO =
+ "YARN_CONTAINER_EXIT_STATUS";
+
+ public static final String STATE_EVENT_INFO =
+ "YARN_CONTAINER_STATE";
+
+}