You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ro...@apache.org on 2019/02/07 04:37:00 UTC
[hadoop] 02/02: YARN-8219. Add application launch time to ATSV2.
Contributed by Abhishek Modi.
This is an automated email from the ASF dual-hosted git repository.
rohithsharmaks pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 7fa62e150ce86f7cb0e88d593b8f07e3384a9a60
Author: Rohith Sharma K S <ro...@apache.org>
AuthorDate: Wed Feb 6 09:31:20 2019 +0530
YARN-8219. Add application launch time to ATSV2. Contributed by Abhishek Modi.
---
.../yarn/server/metrics/ApplicationMetricsConstants.java | 3 +++
.../yarn/util/timeline/TimelineEntityV2Converter.java | 9 +++++++--
.../metrics/AbstractSystemMetricsPublisher.java | 5 +++++
.../metrics/CombinedSystemMetricsPublisher.java | 6 ++++++
.../resourcemanager/metrics/NoOpSystemMetricPublisher.java | 4 ++++
.../resourcemanager/metrics/SystemMetricsPublisher.java | 2 ++
.../metrics/TimelineServiceV2Publisher.java | 14 ++++++++++++++
.../yarn/server/resourcemanager/rmapp/RMAppImpl.java | 4 +++-
.../metrics/TestSystemMetricsPublisherForV2.java | 4 +++-
9 files changed, 47 insertions(+), 4 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 93da565..6428fa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -34,6 +34,9 @@ public class ApplicationMetricsConstants {
public static final String FINISHED_EVENT_TYPE =
"YARN_APPLICATION_FINISHED";
+ public static final String LAUNCHED_EVENT_TYPE =
+ "YARN_APPLICATION_LAUNCHED";
+
public static final String ACLS_UPDATED_EVENT_TYPE =
"YARN_APPLICATION_ACLS_UPDATED";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
index 98e1e7a..9fefc09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineEntityV2Converter.java
@@ -258,6 +258,7 @@ public final class TimelineEntityV2Converter {
String type = null;
boolean unmanagedApplication = false;
long createdTime = 0;
+ long launchTime = 0;
long finishedTime = 0;
float progress = 0.0f;
int applicationPriority = 0;
@@ -417,6 +418,9 @@ public final class TimelineEntityV2Converter {
ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
createdTime = event.getTimestamp();
} else if (event.getId().equals(
+ ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE)) {
+ launchTime = event.getTimestamp();
+ } else if (event.getId().equals(
ApplicationMetricsConstants.UPDATED_EVENT_TYPE)) {
// This type of events are parsed in time-stamp descending order
// which means the previous event could override the information
@@ -449,8 +453,9 @@ public final class TimelineEntityV2Converter {
return ApplicationReport.newInstance(
ApplicationId.fromString(entity.getId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
- diagnosticsInfo, null, createdTime, finishedTime, finalStatus,
- appResources, null, progress, type, null, appTags, unmanagedApplication,
+ diagnosticsInfo, null, createdTime, launchTime,
+ finishedTime, finalStatus, appResources, null,
+ progress, type, null, appTags, unmanagedApplication,
Priority.newInstance(applicationPriority), appNodeLabelExpression,
amNodeLabelExpression);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
index 96b153f..09f44d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
/**
* Abstract implementation of SystemMetricsPublisher which is then extended by
@@ -122,6 +123,10 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService
PUBLISH_ENTITY, PUBLISH_APPLICATION_FINISHED_ENTITY
}
+ @Override
+ public void appLaunched(RMApp app, long launchTime) {
+ }
+
/**
* TimelinePublishEvent's hash code should be based on application's id this
* will ensure all the events related to a particular app goes to particular
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java
index 9646747..8e04a1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/CombinedSystemMetricsPublisher.java
@@ -45,6 +45,12 @@ public class CombinedSystemMetricsPublisher implements SystemMetricsPublisher {
}
}
+ @Override public void appLaunched(RMApp app, long launchTime) {
+ for (SystemMetricsPublisher publisher : this.publishers) {
+ publisher.appLaunched(app, launchTime);
+ }
+ }
+
@Override
public void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime) {
for (SystemMetricsPublisher publisher : this.publishers) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
index e0bf3c7..537c88c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
@@ -67,4 +67,8 @@ public class NoOpSystemMetricPublisher implements SystemMetricsPublisher{
public void appStateUpdated(RMApp app, YarnApplicationState appState,
long updatedTime) {
}
+
+ @Override
+ public void appLaunched(RMApp app, long launchTime) {
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index eeea434..137d790 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -32,6 +32,8 @@ public interface SystemMetricsPublisher {
void appCreated(RMApp app, long createdTime);
+ void appLaunched(RMApp app, long launchTime);
+
void appACLsUpdated(RMApp app, String appViewACLs, long updatedTime);
void appUpdated(RMApp app, long updatedTime);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 95d75b2..e93c26d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -155,6 +155,20 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
@SuppressWarnings("unchecked")
@Override
+ public void appLaunched(RMApp app, long launchTime) {
+ ApplicationEntity entity =
+ createApplicationEntity(app.getApplicationId());
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ApplicationMetricsConstants.LAUNCHED_EVENT_TYPE);
+ tEvent.setTimestamp(launchTime);
+ entity.addEvent(tEvent);
+
+ getDispatcher().getEventHandler().handle(new TimelineV2PublishEvent(
+ SystemMetricsEventType.PUBLISH_ENTITY, entity, app.getApplicationId()));
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
public void appFinished(RMApp app, RMAppState state, long finishedTime) {
ApplicationEntity entity = createApplicationEntity(app.getApplicationId());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index bd645d7..696b39d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -293,7 +293,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition())
- // Handle AppAttemptLaunch to upate the launchTime and publish to ATS
+ // Handle AppAttemptLaunch to update the launchTime and publish to ATS
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.ATTEMPT_LAUNCHED,
new AttemptLaunchedTransition())
@@ -1067,6 +1067,8 @@ public class RMAppImpl implements RMApp, Recoverable {
app.getCurrentAppAttempt().getAppAttemptId()+
"launchTime: "+event.getTimestamp());
app.launchTime = event.getTimestamp();
+ app.rmContext.getSystemMetricsPublisher().appLaunched(
+ app, app.launchTime);
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 3b503e5..ca3f47d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -203,6 +203,7 @@ public class TestSystemMetricsPublisherForV2 {
RMApp app = createAppAndRegister(appId);
metricsPublisher.appCreated(app, app.getStartTime());
+ metricsPublisher.appLaunched(app, app.getLaunchTime());
metricsPublisher.appACLsUpdated(app, "user1,user2", 4L);
metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
dispatcher.await();
@@ -221,7 +222,7 @@ public class TestSystemMetricsPublisherForV2 {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(
- appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
+ appFile, 4, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
}
@Test(timeout = 10000)
@@ -356,6 +357,7 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getQueue()).thenReturn("test queue");
when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
+ when(app.getLaunchTime()).thenReturn(Integer.MAX_VALUE + 2L);
when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
when(app.getDiagnostics()).thenReturn(
new StringBuilder("test diagnostics info"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org