You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2019/01/29 19:20:48 UTC
[hadoop] branch branch-2.7 updated: YARN-6616: YARN AHS shows
submitTime for jobs same as startTime. Contributed by Prabhu Joseph
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-2.7
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.7 by this push:
new 17638c1 YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph
17638c1 is described below
commit 17638c15e9dc8a1d326db8a96016ba84968909b0
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Tue Jan 29 19:12:55 2019 +0000
YARN-6616: YARN AHS shows submitTime for jobs same as startTime. Contributed by Prabhu Joseph
---
.../hadoop/yarn/api/records/ApplicationReport.java | 31 +++++++++++++++++++++-
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/client/api/impl/TestAHSClient.java | 4 ++-
.../records/impl/pb/ApplicationReportPBImpl.java | 12 +++++++++
.../ApplicationHistoryManagerImpl.java | 6 ++---
.../ApplicationHistoryManagerOnTimelineStore.java | 15 ++++++++---
...stApplicationHistoryManagerOnTimelineStore.java | 1 +
.../webapp/TestAHSWebServices.java | 1 +
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 2 +-
.../applicationsmanager/MockAsm.java | 16 +++++++----
10 files changed, 74 insertions(+), 15 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
index ff4fb52..d79d54d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java
@@ -57,7 +57,8 @@ public abstract class ApplicationReport {
ApplicationAttemptId applicationAttemptId, String user, String queue,
String name, String host, int rpcPort, Token clientToAMToken,
YarnApplicationState state, String diagnostics, String url,
- long startTime, long finishTime, FinalApplicationStatus finalStatus,
+ long startTime, long submitTime, long finishTime,
+ FinalApplicationStatus finalStatus,
ApplicationResourceUsageReport appResources, String origTrackingUrl,
float progress, String applicationType, Token amRmToken) {
ApplicationReport report = Records.newRecord(ApplicationReport.class);
@@ -73,6 +74,7 @@ public abstract class ApplicationReport {
report.setDiagnostics(diagnostics);
report.setTrackingUrl(url);
report.setStartTime(startTime);
+ report.setSubmitTime(submitTime);
report.setFinishTime(finishTime);
report.setFinalApplicationStatus(finalStatus);
report.setApplicationResourceUsageReport(appResources);
@@ -83,6 +85,21 @@ public abstract class ApplicationReport {
return report;
}
+ @Private
+ @Unstable
+ public static ApplicationReport newInstance(ApplicationId applicationId,
+ ApplicationAttemptId applicationAttemptId, String user, String queue,
+ String name, String host, int rpcPort, Token clientToAMToken,
+ YarnApplicationState state, String diagnostics, String url,
+ long startTime, long finishTime, FinalApplicationStatus finalStatus,
+ ApplicationResourceUsageReport appResources, String origTrackingUrl,
+ float progress, String applicationType, Token amRmToken) {
+ return newInstance(applicationId, applicationAttemptId, user, queue, name,
+ host, rpcPort, clientToAMToken, state, diagnostics, url, startTime,
+ startTime, finishTime, finalStatus, appResources, origTrackingUrl,
+ progress, applicationType, amRmToken);
+ }
+
/**
* Get the <code>ApplicationId</code> of the application.
* @return <code>ApplicationId</code> of the application
@@ -258,6 +275,18 @@ public abstract class ApplicationReport {
public abstract void setStartTime(long startTime);
/**
+ * Get the <em>submit time</em> of the application.
+ * @return <em>submit time</em> of the application
+ */
+ @Public
+ @Stable
+ public abstract long getSubmitTime();
+
+ @Private
+ @Unstable
+ public abstract void setSubmitTime(long submitTime);
+
+ /**
* Get the <em>finish time</em> of the application.
* @return <em>finish time</em> of the application
*/
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 2edff99..5d075c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -194,6 +194,7 @@ message ApplicationReportProto {
optional string applicationType = 18;
optional hadoop.common.TokenProto am_rm_token = 19;
repeated string applicationTags = 20;
+ optional int64 submitTime = 21;
}
message ApplicationAttemptReportProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index c3e3c41..3a8b7b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -102,6 +102,8 @@ public class TestAHSClient {
Assert.assertEquals(report, expectedReports.get(0));
Assert.assertEquals(report.getApplicationId().toString(), expectedReports
.get(0).getApplicationId().toString());
+ Assert.assertEquals(report.getSubmitTime(), expectedReports.get(0)
+ .getSubmitTime());
client.stop();
}
@@ -332,7 +334,7 @@ public class TestAHSClient {
ApplicationReport.newInstance(applicationId,
ApplicationAttemptId.newInstance(applicationId, 1), "user",
"queue", "appname", "host", 124, null,
- YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
+ YarnApplicationState.RUNNING, "diagnostics", "url", 1, 2, 3,
FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
null);
List<ApplicationReport> applicationReports =
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
index dd3e2bc..24b1551 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
@@ -207,6 +207,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
}
@Override
+ public long getSubmitTime() {
+ ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getSubmitTime();
+ }
+
+ @Override
public long getFinishTime() {
ApplicationReportProtoOrBuilder p = viaProto ? proto : builder;
return p.getFinishTime();
@@ -400,6 +406,12 @@ public class ApplicationReportPBImpl extends ApplicationReport {
}
@Override
+ public void setSubmitTime(long submitTime) {
+ maybeInitBuilder();
+ builder.setSubmitTime(submitTime);
+ }
+
+ @Override
public void setFinishTime(long finishTime) {
maybeInitBuilder();
builder.setFinishTime(finishTime);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 479858f..aadb0fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -137,9 +137,9 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
currentApplicationAttemptId, appHistory.getUser(), appHistory.getQueue(),
appHistory.getApplicationName(), host, rpcPort, null,
appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
- trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
- appHistory.getFinalApplicationStatus(), null, "", 100,
- appHistory.getApplicationType(), null);
+ trackingUrl, appHistory.getStartTime(), appHistory.getSubmitTime(),
+ appHistory.getFinishTime(), appHistory.getFinalApplicationStatus(),
+ null, "", 100, appHistory.getApplicationType(), null);
}
private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index c177112..0cec43a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -242,6 +242,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
String name = null;
String type = null;
long createdTime = 0;
+ long submittedTime = 0;
long finishedTime = 0;
ApplicationAttemptId latestApplicationAttemptId = null;
String diagnosticsInfo = null;
@@ -268,8 +269,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
return new ApplicationReportExt(ApplicationReport.newInstance(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
- diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
- null, 1.0F, type, null), appViewACLs);
+ diagnosticsInfo, null, createdTime, submittedTime, finishedTime,
+ finalStatus, null, null, 1.0F, type, null), appViewACLs);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
queue =
@@ -286,6 +287,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
entityInfo.get(ApplicationMetricsConstants.TYPE_ENTITY_INFO)
.toString();
}
+ if (entityInfo.containsKey(ApplicationMetricsConstants.
+ SUBMITTED_TIME_ENTITY_INFO)) {
+ submittedTime = Long.parseLong(entityInfo.get(
+ ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO)
+ .toString());
+ }
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
long vcoreSeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
@@ -344,8 +351,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
return new ApplicationReportExt(ApplicationReport.newInstance(
ConverterUtils.toApplicationId(entity.getEntityId()),
latestApplicationAttemptId, user, queue, name, null, -1, null, state,
- diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
- null, 1.0F, type, null), appViewACLs);
+ diagnosticsInfo, null, createdTime, submittedTime, finishedTime,
+ finalStatus, appResources, null, 1.0F, type, null), appViewACLs);
}
private static ApplicationAttemptReport convertToApplicationAttemptReport(
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 8672953..664b85d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -182,6 +182,7 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Assert.assertEquals("test queue", app.getQueue());
Assert.assertEquals(Integer.MAX_VALUE + 2L
+ app.getApplicationId().getId(), app.getStartTime());
+ Assert.assertEquals(Integer.MAX_VALUE + 1L, app.getSubmitTime());
Assert.assertEquals(Integer.MAX_VALUE + 3L
+ +app.getApplicationId().getId(), app.getFinishTime());
Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
index 80ffa03..4329b6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestAHSWebServices.java
@@ -311,6 +311,7 @@ public class TestAHSWebServices extends JerseyTestBase {
assertEquals("test app", app.get("name"));
assertEquals(round == 0 ? "test diagnostics info" : "",
app.get("diagnosticsInfo"));
+ assertEquals(Integer.MAX_VALUE + 1L, app.get("submittedTime"));
assertEquals("test queue", app.get("queue"));
assertEquals("user1", app.get("user"));
assertEquals("test app type", app.get("type"));
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index 0e17e3d..de14e03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -77,7 +77,7 @@ public class AppInfo {
diagnosticsInfo = app.getDiagnostics();
trackingUrl = app.getTrackingUrl();
originalTrackingUrl = app.getOriginalTrackingUrl();
- submittedTime = app.getStartTime();
+ submittedTime = app.getSubmitTime();
startedTime = app.getStartTime();
finishedTime = app.getFinishTime();
elapsedTime = Times.elapsed(startedTime, finishedTime);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index f8d92aa..6ef8d1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -203,6 +203,7 @@ public abstract class MockAsm extends MockApps {
final String name = newAppName();
final String queue = newQueue();
final long start = 123456 + i * 1000;
+ final long submit = start + i * 50;
final long finish = 234567 + i * 1000;
final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
YarnApplicationState[] allStates = YarnApplicationState.values();
@@ -239,6 +240,11 @@ public abstract class MockAsm extends MockApps {
}
@Override
+ public long getSubmitTime() {
+ return submit;
+ }
+
+ @Override
public long getFinishTime() {
return finish;
}
@@ -282,13 +288,13 @@ public abstract class MockAsm extends MockApps {
public ApplicationReport createAndGetApplicationReport(
String clientUserName, boolean allowAccess) {
ApplicationResourceUsageReport usageReport =
- ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
+ ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
0, 0);
ApplicationReport report = ApplicationReport.newInstance(
- getApplicationId(), appAttemptId, getUser(), getQueue(),
- getName(), null, 0, null, null, getDiagnostics().toString(),
- getTrackingUrl(), getStartTime(), getFinishTime(),
- getFinalApplicationStatus(), usageReport , null, getProgress(),
+ getApplicationId(), appAttemptId, getUser(), getQueue(),
+ getName(), null, 0, null, null, getDiagnostics().toString(),
+ getTrackingUrl(), getStartTime(), getSubmitTime(), getFinishTime(),
+ getFinalApplicationStatus(), usageReport, null, getProgress(),
type, null);
return report;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org