You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2016/11/10 22:37:46 UTC
hadoop git commit: YARN-4218. Metric for resource*time that was
preempted. Contributed by Chang Li.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3a9841953 -> 93eeb1316
YARN-4218. Metric for resource*time that was preempted. Contributed by Chang Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93eeb131
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93eeb131
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93eeb131
Branch: refs/heads/trunk
Commit: 93eeb13164707d0e3556c2bf737bd2ee09a335c6
Parents: 3a98419
Author: Eric Payne <ep...@apache.org>
Authored: Thu Nov 10 22:35:12 2016 +0000
Committer: Eric Payne <ep...@apache.org>
Committed: Thu Nov 10 22:35:12 2016 +0000
----------------------------------------------------------------------
.../records/ApplicationResourceUsageReport.java | 43 +++++++++++++++++++-
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 7 ++++
.../hadoop/yarn/client/cli/TestYarnCLI.java | 4 +-
.../ApplicationResourceUsageReportPBImpl.java | 28 +++++++++++++
...pplicationHistoryManagerOnTimelineStore.java | 8 +++-
...pplicationHistoryManagerOnTimelineStore.java | 11 ++++-
.../metrics/ApplicationMetricsConstants.java | 6 +++
.../hadoop/yarn/server/utils/BuilderUtils.java | 5 ++-
.../server/resourcemanager/RMAppManager.java | 2 +
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../metrics/TimelineServiceV1Publisher.java | 4 ++
.../metrics/TimelineServiceV2Publisher.java | 6 +++
.../resourcemanager/recovery/RMStateStore.java | 10 +++--
.../records/ApplicationAttemptStateData.java | 39 ++++++++++++++++--
.../pb/ApplicationAttemptStateDataPBImpl.java | 24 +++++++++++
.../server/resourcemanager/rmapp/RMAppImpl.java | 13 +++++-
.../resourcemanager/rmapp/RMAppMetrics.java | 16 +++++++-
.../rmapp/attempt/RMAppAttemptImpl.java | 11 ++++-
.../rmapp/attempt/RMAppAttemptMetrics.java | 16 ++++++++
.../rmcontainer/RMContainerImpl.java | 16 ++++----
.../scheduler/SchedulerApplicationAttempt.java | 2 +-
.../resourcemanager/webapp/RMAppBlock.java | 8 +++-
.../resourcemanager/webapp/dao/AppInfo.java | 11 +++++
.../yarn_server_resourcemanager_recovery.proto | 2 +
.../server/resourcemanager/TestAppManager.java | 3 +-
.../applicationsmanager/MockAsm.java | 4 +-
.../metrics/TestSystemMetricsPublisher.java | 13 +++++-
.../TestSystemMetricsPublisherForV2.java | 4 +-
.../recovery/RMStateStoreTestBase.java | 4 +-
.../recovery/TestZKRMStateStore.java | 2 +-
.../resourcemanager/webapp/TestAppPage.java | 3 +-
.../webapp/TestRMWebAppFairScheduler.java | 3 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
34 files changed, 297 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index 34efee8..3cf8f3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,7 +36,8 @@ public abstract class ApplicationResourceUsageReport {
public static ApplicationResourceUsageReport newInstance(
int numUsedContainers, int numReservedContainers, Resource usedResources,
Resource reservedResources, Resource neededResources, long memorySeconds,
- long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc) {
+ long vcoreSeconds, float queueUsagePerc, float clusterUsagePerc,
+ long preemptedMemorySeconds, long preemptedVcoresSeconds) {
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@@ -48,6 +49,8 @@ public abstract class ApplicationResourceUsageReport {
report.setVcoreSeconds(vcoreSeconds);
report.setQueueUsagePercentage(queueUsagePerc);
report.setClusterUsagePercentage(clusterUsagePerc);
+ report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+ report.setPreemptedVcoreSeconds(preemptedVcoresSeconds);
return report;
}
@@ -188,4 +191,42 @@ public abstract class ApplicationResourceUsageReport {
@Private
@Unstable
public abstract void setClusterUsagePercentage(float clusterUsagePerc);
+
+ /**
+ * Set the aggregated amount of memory preempted (in megabytes)
+ * the application has allocated times the number of seconds
+ * the application has been running.
+ * @param memorySeconds the aggregated amount of memory seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setPreemptedMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the aggregated amount of memory preempted(in megabytes)
+ * the application has allocated times the number of
+ * seconds the application has been running.
+ * @return the aggregated amount of memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getPreemptedMemorySeconds();
+
+ /**
+ * Set the aggregated number of vcores preempted that the application has
+ * allocated times the number of seconds the application has been running.
+ * @param vcoreSeconds the aggregated number of vcore seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
+
+ /**
+ * Get the aggregated number of vcores preempted that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getPreemptedVcoreSeconds();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index b59d02b..1022a38 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -184,6 +184,8 @@ message ApplicationResourceUsageReportProto {
optional int64 vcore_seconds = 7;
optional float queue_usage_percentage = 8;
optional float cluster_usage_percentage = 9;
+ optional int64 preempted_memory_seconds = 10;
+ optional int64 preempted_vcore_seconds = 11;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 21df6a2..b841601 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -658,8 +658,15 @@ public class ApplicationCLI extends YarnCLI {
//completed app report in the timeline server doesn't have usage report
appReportStr.print(usageReport.getMemorySeconds() + " MB-seconds, ");
appReportStr.println(usageReport.getVcoreSeconds() + " vcore-seconds");
+ appReportStr.print("\tAggregate Resource Preempted : ");
+ appReportStr.print(usageReport.getPreemptedMemorySeconds() +
+ " MB-seconds, ");
+ appReportStr.println(usageReport.getPreemptedVcoreSeconds() +
+ " vcore-seconds");
} else {
appReportStr.println("N/A");
+ appReportStr.print("\tAggregate Resource Preempted : ");
+ appReportStr.println("N/A");
}
appReportStr.print("\tLog Aggregation Status : ");
appReportStr.println(appReport.getLogAggregationStatus() == null ? "N/A"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index c1cabb0..09dec89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -115,7 +115,7 @@ public class TestYarnCLI {
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport.newInstance(
- 2, 0, null, null, null, 123456, 4567, 0, 0);
+ 2, 0, null, null, null, 123456, 4567, 0, 0, 1111, 2222);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
@@ -148,6 +148,8 @@ public class TestYarnCLI {
pw.println("\tAM Host : host");
pw.println("\tAggregate Resource Allocation : " +
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
+ pw.println("\tAggregate Resource Preempted : " +
+ (i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds"));
pw.println("\tLog Aggregation Status : SUCCEEDED");
pw.println("\tDiagnostics : diagnostics");
pw.println("\tUnmanaged Application : false");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index 5b1b85e..b6a38e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -224,6 +224,34 @@ extends ApplicationResourceUsageReport {
return (p.getVcoreSeconds());
}
+ @Override
+ public synchronized void setPreemptedMemorySeconds(
+ long preemptedMemorySeconds) {
+ maybeInitBuilder();
+ builder.setPreemptedMemorySeconds(preemptedMemorySeconds);
+ }
+
+ @Override
+ public synchronized long getPreemptedMemorySeconds() {
+ ApplicationResourceUsageReportProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getPreemptedMemorySeconds();
+ }
+
+ @Override
+ public synchronized void setPreemptedVcoreSeconds(
+ long vcoreSeconds) {
+ maybeInitBuilder();
+ builder.setPreemptedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
+ public synchronized long getPreemptedVcoreSeconds() {
+ ApplicationResourceUsageReportProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return (p.getPreemptedVcoreSeconds());
+ }
+
private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
return new ResourcePBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 6e6576a..3249c8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -334,9 +334,15 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
ApplicationMetricsConstants.APP_CPU_METRICS).toString());
long memorySeconds=Long.parseLong(entityInfo.get(
ApplicationMetricsConstants.APP_MEM_METRICS).toString());
+ long preemptedMemorySeconds = Long.parseLong(entityInfo.get(
+ ApplicationMetricsConstants
+ .APP_MEM_PREEMPT_METRICS).toString());
+ long preemptedVcoreSeconds = Long.parseLong(entityInfo.get(
+ ApplicationMetricsConstants
+ .APP_CPU_PREEMPT_METRICS).toString());
appResources = ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds, 0,
- 0);
+ 0, preemptedMemorySeconds, preemptedVcoreSeconds);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
appTags = new HashSet<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index 526cb8f..3d4e91b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -245,6 +245,11 @@ public class TestApplicationHistoryManagerOnTimelineStore {
applicationResourceUsageReport.getMemorySeconds());
Assert
.assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
+ Assert.assertEquals(456,
+ applicationResourceUsageReport.getPreemptedMemorySeconds());
+ Assert
+ .assertEquals(789, applicationResourceUsageReport
+ .getPreemptedVcoreSeconds());
Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
app.getFinalApplicationStatus());
Assert.assertEquals(YarnApplicationState.FINISHED,
@@ -503,8 +508,10 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS,123);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS,345);
+ entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
+ entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
+ entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,456);
+ entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,789);
if (emptyACLs) {
entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO, "");
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 521e0af..4cec409 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -88,6 +88,12 @@ public class ApplicationMetricsConstants {
public static final String APP_AM_CONTAINER_PREEMPTED =
"YARN_APPLICATION_AM_CONTAINER_PREEMPTED";
+ public static final String APP_CPU_PREEMPT_METRICS =
+ "YARN_APPLICATION_CPU_PREEMPT_METRIC";
+
+ public static final String APP_MEM_PREEMPT_METRICS =
+ "YARN_APPLICATION_MEM_PREEMPT_METRIC";
+
public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
"YARN_APPLICATION_LATEST_APP_ATTEMPT";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 50df12e..74c06ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -446,7 +446,8 @@ public class BuilderUtils {
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
int numUsedContainers, int numReservedContainers, Resource usedResources,
Resource reservedResources, Resource neededResources, long memorySeconds,
- long vcoreSeconds) {
+ long vcoreSeconds, long preemptedMemorySeconds,
+ long preemptedVcoreSeconds) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
@@ -456,6 +457,8 @@ public class BuilderUtils {
report.setNeededResources(neededResources);
report.setMemorySeconds(memorySeconds);
report.setVcoreSeconds(vcoreSeconds);
+ report.setPreemptedMemorySeconds(preemptedMemorySeconds);
+ report.setPreemptedVcoreSeconds(preemptedVcoreSeconds);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 7144421..e566243 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -182,6 +182,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("finalStatus", app.getFinalApplicationStatus())
.add("memorySeconds", metrics.getMemorySeconds())
.add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds())
+ .add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds())
.add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
.add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
.add("preemptedResources", metrics.getResourcePreempted())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 7e31e70..98a34d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -453,7 +453,7 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
- Resources.createResource(-1, -1), 0, 0);
+ Resources.createResource(-1, -1), 0, 0, 0, 0);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index cbf6a73..aa3ef0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -144,6 +144,10 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
appMetrics.getVcoreSeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
appMetrics.getMemorySeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,
+ appMetrics.getPreemptedMemorySeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,
+ appMetrics.getPreemptedVcoreSeconds());
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index f039ebe..6eb6eb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -194,6 +194,12 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
ApplicationMetricsConstants.APP_MEM_METRICS, timestamp,
appMetrics.getMemorySeconds()));
entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp,
+ appMetrics.getPreemptedMemorySeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, timestamp,
+ appMetrics.getPreemptedVcoreSeconds()));
+ entityMetrics.add(getTimelineMetric(
ApplicationMetricsConstants.APP_RESOURCE_PREEMPTED_CPU, timestamp,
appMetrics.getResourcePreempted().getVirtualCores()));
entityMetrics.add(getTimelineMetric(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index d1f8b40..fc12522 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppR
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
@@ -826,16 +827,19 @@ public abstract class RMStateStore extends AbstractService {
*/
public void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
-
+ RMAppAttemptMetrics attempMetrics = appAttempt.getRMAppAttemptMetrics();
AggregateAppResourceUsage resUsage =
- appAttempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
+ attempMetrics.getAggregateAppResourceUsage();
ApplicationAttemptStateData attemptState =
ApplicationAttemptStateData.newInstance(
appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(),
credentials, appAttempt.getStartTime(),
resUsage.getMemorySeconds(),
- resUsage.getVcoreSeconds());
+ resUsage.getVcoreSeconds(),
+ attempMetrics.getPreemptedMemory(),
+ attempMetrics.getPreemptedVcore()
+ );
dispatcher.getEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
index 391783b..67aaf94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
@@ -40,7 +40,8 @@ public abstract class ApplicationAttemptStateData {
Credentials attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
- long finishTime, long memorySeconds, long vcoreSeconds) {
+ long finishTime, long memorySeconds, long vcoreSeconds,
+ long preemptedMemorySeconds, long preemptedVcoreSeconds) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@@ -55,16 +56,20 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setFinishTime(finishTime);
attemptStateData.setMemorySeconds(memorySeconds);
attemptStateData.setVcoreSeconds(vcoreSeconds);
+ attemptStateData.setPreemptedMemorySeconds(preemptedMemorySeconds);
+ attemptStateData.setPreemptedVcoreSeconds(preemptedVcoreSeconds);
return attemptStateData;
}
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container masterContainer,
Credentials attemptTokens, long startTime, long memorySeconds,
- long vcoreSeconds) {
+ long vcoreSeconds, long preemptedMemorySeconds,
+ long preemptedVcoreSeconds) {
return newInstance(attemptId, masterContainer, attemptTokens,
startTime, null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
- memorySeconds, vcoreSeconds);
+ memorySeconds, vcoreSeconds,
+ preemptedMemorySeconds, preemptedVcoreSeconds);
}
@@ -182,4 +187,32 @@ public abstract class ApplicationAttemptStateData {
@Public
@Unstable
public abstract void setVcoreSeconds(long vcoreSeconds);
+
+ /**
+ * Get the <em>preempted memory seconds</em>
+ * (in MB seconds) of the application.
+ * @return <em>preempted memory seconds</em>
+ * (in MB seconds) of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getPreemptedMemorySeconds();
+
+ @Public
+ @Unstable
+ public abstract void setPreemptedMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the <em>preempted vcore seconds</em>
+ * of the application.
+ * @return <em>preempted vcore seconds</em>
+ * of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getPreemptedVcoreSeconds();
+
+ @Public
+ @Unstable
+ public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index bae3f9c..e89726f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -263,6 +263,30 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ public long getPreemptedMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getPreemptedMemorySeconds();
+ }
+
+ @Override
+ public long getPreemptedVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getPreemptedVcoreSeconds();
+ }
+
+ @Override
+ public void setPreemptedMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setPreemptedMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public void setPreemptedVcoreSeconds(long vcoreSeconds) {
+ maybeInitBuilder();
+ builder.setPreemptedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
public FinalApplicationStatus getFinalApplicationStatus() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasFinalApplicationStatus()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 74e641d..a376311 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -757,6 +757,12 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppMetrics rmAppMetrics = getRMAppMetrics();
appUsageReport.setMemorySeconds(rmAppMetrics.getMemorySeconds());
appUsageReport.setVcoreSeconds(rmAppMetrics.getVcoreSeconds());
+ appUsageReport.
+ setPreemptedMemorySeconds(rmAppMetrics.
+ getPreemptedMemorySeconds());
+ appUsageReport.
+ setPreemptedVcoreSeconds(rmAppMetrics.
+ getPreemptedVcoreSeconds());
}
if (currentApplicationAttemptId == null) {
@@ -1630,6 +1636,8 @@ public class RMAppImpl implements RMApp, Recoverable {
int numNonAMContainerPreempted = 0;
long memorySeconds = 0;
long vcoreSeconds = 0;
+ long preemptedMemorySeconds = 0;
+ long preemptedVcoreSeconds = 0;
for (RMAppAttempt attempt : attempts.values()) {
if (null != attempt) {
RMAppAttemptMetrics attemptMetrics =
@@ -1645,12 +1653,15 @@ public class RMAppImpl implements RMApp, Recoverable {
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
memorySeconds += resUsage.getMemorySeconds();
vcoreSeconds += resUsage.getVcoreSeconds();
+ preemptedMemorySeconds += attemptMetrics.getPreemptedMemory();
+ preemptedVcoreSeconds += attemptMetrics.getPreemptedVcore();
}
}
return new RMAppMetrics(resourcePreempted,
numNonAMContainerPreempted, numAMContainerPreempted,
- memorySeconds, vcoreSeconds);
+ memorySeconds, vcoreSeconds,
+ preemptedMemorySeconds, preemptedVcoreSeconds);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 5091470..fa068ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -26,15 +26,20 @@ public class RMAppMetrics {
final int numAMContainersPreempted;
final long memorySeconds;
final long vcoreSeconds;
+ private final long preemptedMemorySeconds;
+ private final long preemptedVcoreSeconds;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
- long memorySeconds, long vcoreSeconds) {
+ long memorySeconds, long vcoreSeconds, long preemptedMemorySeconds,
+ long preemptedVcoreSeconds) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
this.memorySeconds = memorySeconds;
this.vcoreSeconds = vcoreSeconds;
+ this.preemptedMemorySeconds = preemptedMemorySeconds;
+ this.preemptedVcoreSeconds = preemptedVcoreSeconds;
}
public Resource getResourcePreempted() {
@@ -56,4 +61,13 @@ public class RMAppMetrics {
public long getVcoreSeconds() {
return vcoreSeconds;
}
+
+ public long getPreemptedMemorySeconds() {
+ return preemptedMemorySeconds;
+ }
+
+ public long getPreemptedVcoreSeconds() {
+ return preemptedVcoreSeconds;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 5b78eb8..e3d1855 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -889,6 +889,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.attemptMetrics.getAggregateAppResourceUsage();
report.setMemorySeconds(resUsage.getMemorySeconds());
report.setVcoreSeconds(resUsage.getVcoreSeconds());
+ report.setPreemptedMemorySeconds(
+ this.attemptMetrics.getPreemptedMemory());
+ report.setPreemptedVcoreSeconds(
+ this.attemptMetrics.getPreemptedVcore());
return report;
} finally {
this.readLock.unlock();
@@ -921,6 +925,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finishTime = attemptState.getFinishTime();
this.attemptMetrics.updateAggregateAppResourceUsage(
attemptState.getMemorySeconds(),attemptState.getVcoreSeconds());
+ this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
+ attemptState.getPreemptedMemorySeconds(),
+ attemptState.getPreemptedVcoreSeconds());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1296,7 +1303,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
startTime, stateToBeStored, finalTrackingUrl, diags,
finalStatus, exitStatus,
getFinishTime(), resUsage.getMemorySeconds(),
- resUsage.getVcoreSeconds());
+ resUsage.getVcoreSeconds(),
+ this.attemptMetrics.getPreemptedMemory(),
+ this.attemptMetrics.getPreemptedVcore());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 0a3638b..a642e45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -50,6 +50,8 @@ public class RMAppAttemptMetrics {
private WriteLock writeLock;
private AtomicLong finishedMemorySeconds = new AtomicLong(0);
private AtomicLong finishedVcoreSeconds = new AtomicLong(0);
+ private AtomicLong preemptedMemorySeconds = new AtomicLong(0);
+ private AtomicLong preemptedVcoreSeconds = new AtomicLong(0);
private RMContext rmContext;
private int[][] localityStatistics =
@@ -98,6 +100,14 @@ public class RMAppAttemptMetrics {
}
}
+ public long getPreemptedMemory() {
+ return preemptedMemorySeconds.get();
+ }
+
+ public long getPreemptedVcore() {
+ return preemptedVcoreSeconds.get();
+ }
+
public int getNumNonAMContainersPreempted() {
return numNonAMContainersPreempted.get();
}
@@ -134,6 +144,12 @@ public class RMAppAttemptMetrics {
this.finishedVcoreSeconds.addAndGet(finishedVcoreSeconds);
}
+ public void updateAggregatePreemptedAppResourceUsage(
+ long preemptedMemorySeconds, long preemptedVcoreSeconds) {
+ this.preemptedMemorySeconds.addAndGet(preemptedMemorySeconds);
+ this.preemptedVcoreSeconds.addAndGet(preemptedVcoreSeconds);
+ }
+
public void incNumAllocatedContainers(NodeType containerType,
NodeType requestType) {
localityStatistics[containerType.index][requestType.index]++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 4294ef0..a651d54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -729,19 +729,12 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
}
private static void updateAttemptMetrics(RMContainerImpl container) {
- // If this is a preempted container, update preemption metrics
Resource resource = container.getContainer().getResource();
RMAppAttempt rmAttempt = container.rmContext.getRMApps()
.get(container.getApplicationAttemptId().getApplicationId())
.getCurrentAppAttempt();
if (rmAttempt != null) {
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
- container);
- }
-
long usedMillis = container.finishTime - container.creationTime;
long memorySeconds = resource.getMemorySize()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
@@ -749,6 +742,15 @@ public class RMContainerImpl implements RMContainer, Comparable<RMContainer> {
* usedMillis / DateUtils.MILLIS_PER_SECOND;
rmAttempt.getRMAppAttemptMetrics()
.updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds);
+ // If this is a preempted container, update preemption metrics
+ if (ContainerExitStatus.PREEMPTED == container.finishedStatus
+ .getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
+ container);
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregatePreemptedAppResourceUsage(memorySeconds,
+ vcoreSeconds);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index bb1d461..bc52816 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -914,7 +914,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
Resources.add(usedResourceClone, reservedResourceClone),
runningResourceUsage.getMemorySeconds(),
runningResourceUsage.getVcoreSeconds(), queueUsagePerc,
- clusterUsagePerc);
+ clusterUsagePerc, 0, 0);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index cfa21dd..e5d6c16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -101,7 +101,13 @@ public class RMAppBlock extends AppBlock{
._("Aggregate Resource Allocation:",
String.format("%d MB-seconds, %d vcore-seconds",
appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
- appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
+ appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()))
+ ._("Aggregate Preempted Resource Allocation:",
+ String.format("%d MB-seconds, %d vcore-seconds",
+ appMetrics == null ? "N/A" : appMetrics.getPreemptedMemorySeconds(),
+ appMetrics == null ? "N/A" :
+ appMetrics.getPreemptedVcoreSeconds()));
+
pdiv._();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 19cbe43..42449b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -100,6 +100,8 @@ public class AppInfo {
protected long preemptedResourceVCores;
protected int numNonAMContainerPreempted;
protected int numAMContainerPreempted;
+ private long preemptedMemorySeconds;
+ private long preemptedVcoreSeconds;
// list of resource requests
@XmlElement(name = "resourceRequests")
@@ -216,6 +218,8 @@ public class AppInfo {
appMetrics.getResourcePreempted().getVirtualCores();
memorySeconds = appMetrics.getMemorySeconds();
vcoreSeconds = appMetrics.getVcoreSeconds();
+ preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
+ preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
unmanagedApplication =
appSubmissionContext.getUnmanagedAM();
appNodeLabelExpression =
@@ -382,6 +386,13 @@ public class AppInfo {
return vcoreSeconds;
}
+ public long getPreemptedMemorySeconds() {
+ return preemptedMemorySeconds;
+ }
+
+ public long getPreemptedVcoreSeconds() {
+ return preemptedVcoreSeconds;
+ }
public List<ResourceRequestInfo> getResourceRequests() {
return this.resourceRequests;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 4693818..247cd21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -85,6 +85,8 @@ message ApplicationAttemptStateDataProto {
optional int64 memory_seconds = 10;
optional int64 vcore_seconds = 11;
optional int64 finish_time = 12;
+ optional int64 preempted_memory_seconds = 13;
+ optional int64 preempted_vcore_seconds = 14;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 3db0d65..981b045 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -689,7 +689,8 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
RMAppMetrics metrics =
- new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
+ new RMAppMetrics(Resource.newInstance(1234, 56),
+ 10, 1, 16384, 64, 0, 0);
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index e5b166d..f21c7c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -194,7 +194,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
- return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
+ return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0);
}
@Override
@@ -328,7 +328,7 @@ public abstract class MockAsm extends MockApps {
String clientUserName, boolean allowAccess) {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
- 0, 0, 0, 0);
+ 0, 0, 0, 0, 0, 0);
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index b38091e..2b1740f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -223,6 +223,16 @@ public class TestSystemMetricsPublisher {
app.getRMAppMetrics().getVcoreSeconds(),
Long.parseLong(entity.getOtherInfo()
.get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getPreemptedMemorySeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getPreemptedVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS)
+ .toString()));
}
Assert.assertEquals("context", entity.getOtherInfo()
.get(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT));
@@ -496,7 +506,8 @@ public class TestSystemMetricsPublisher {
when(app.getFinalApplicationStatus()).thenReturn(
FinalApplicationStatus.UNDEFINED);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
+ new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
+ Integer.MAX_VALUE, Long.MAX_VALUE));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 4a19744..3ab2501 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -211,7 +211,7 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 6);
+ verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8);
}
@Test(timeout = 10000)
@@ -355,7 +355,7 @@ public class TestSystemMetricsPublisherForV2 {
FinalApplicationStatus.UNDEFINED);
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, Integer.MAX_VALUE,
- Long.MAX_VALUE));
+ Long.MAX_VALUE, 0, 0));
when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 758bbae..8544c13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -361,7 +361,7 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), 0, 0);
+ oldAttemptState.getFinishTime(), 0, 0, 0, 0);
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -385,7 +385,7 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), 0, 0);
+ oldAttemptState.getFinishTime(), 0, 0, 0, 0);
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 19d3064..aef0dfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -428,7 +428,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, 0, 0);
+ finishTime, 0, 0, 0, 0);
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index 8c7b14d..a009bc0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -64,7 +64,8 @@ public class TestAppPage {
when(app.getFinishTime()).thenReturn(0L);
when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
- RMAppMetrics appMetrics = new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
+ RMAppMetrics appMetrics = new RMAppMetrics(
+ Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0);
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index 06fa0d4..78fadef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -142,7 +142,8 @@ public class TestRMWebAppFairScheduler {
MockRMApp app = new MockRMApp(i, i, state) {
@Override
public RMAppMetrics getRMAppMetrics() {
- return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0);
+ return new RMAppMetrics(Resource.newInstance(0, 0),
+ 0, 0, 0, 0, 0, 0);
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93eeb131/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index da28754..fffa55b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1419,7 +1419,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- int expectedNumberOfElements = 32;
+ int expectedNumberOfElements = 34;
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org