You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2018/01/24 06:59:45 UTC
[2/3] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index ed71ea2..0243443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends
private Container masterContainer = null;
private ByteBuffer appAttemptTokens = null;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationAttemptStateDataPBImpl() {
builder = ApplicationAttemptStateDataProto.newBuilder();
@@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public long getMemorySeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getMemorySeconds();
+ return getGuaranteedMemorySeconds();
}
@Override
+ @Deprecated
public long getVcoreSeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getVcoreSeconds();
+ return getGuaranteedVcoreSeconds();
}
@Override
+ @Deprecated
public void setMemorySeconds(long memorySeconds) {
- maybeInitBuilder();
- builder.setMemorySeconds(memorySeconds);
+ setGuaranteedMemorySeconds(memorySeconds);
}
@Override
+ @Deprecated
public void setVcoreSeconds(long vcoreSeconds) {
+ setGuaranteedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
+ public long getGuaranteedMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public void setGuaranteedMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public long getGuaranteedVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getVcoreSeconds();
+ }
+
+ @Override
+ public void setGuaranteedVcoreSeconds(long vcoreSeconds) {
maybeInitBuilder();
builder.setVcoreSeconds(vcoreSeconds);
}
@Override
+ public long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
public long getPreemptedMemorySeconds() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getPreemptedMemorySeconds();
@@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ @Deprecated
+ public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ setGuaranteedResourceSecondsMap(resourceSecondsMap);
+ }
+
+ @Override
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
- p.getApplicationResourceUsageMapList());
- return this.resourceSecondsMap;
+ this.guaranteedResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationResourceUsageMapList());
+ return this.guaranteedResourceSecondsMap;
}
@Override
- public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ public void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
maybeInitBuilder();
builder.clearApplicationResourceUsageMap();
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap != null) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
@@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
}
+
+ @Override
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
+ maybeInitBuilder();
+ builder.clearApplicationOpportunisticResourceUsageMap();
+ this.opportunisticResourceSecondsMap = resourceSecondsMap;
+ if (resourceSecondsMap != null) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 38f666b..dc69af2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -449,6 +449,7 @@ public class RMAppImpl implements RMApp, Recoverable {
} else {
this.startTime = startTime;
}
+ LOG.info(applicationId.toString() + " started at " + this.startTime);
this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
@@ -754,10 +755,12 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppMetrics rmAppMetrics = getRMAppMetrics();
- appUsageReport
- .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
+ appUsageReport.setGuaranteedResourceSecondsMap(
+ rmAppMetrics.getGuaranteedResourceSecondsMap());
appUsageReport.setPreemptedResourceSecondsMap(
rmAppMetrics.getPreemptedResourceSecondsMap());
+ appUsageReport.setOpportunisticResourceSecondsMap(
+ rmAppMetrics.getOpportunisticResourceSecondsMap());
}
if (currentApplicationAttemptId == null) {
@@ -1627,8 +1630,9 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2);
+ Map<String, Long> preemptedSecondsMap = new HashMap<>(2);
+ Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2);
this.readLock.lock();
try {
for (RMAppAttempt attempt : attempts.values()) {
@@ -1644,20 +1648,15 @@ public class RMAppImpl implements RMApp, Recoverable {
// for both running and finished containers.
AggregateAppResourceUsage resUsage =
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
- for (Map.Entry<String, Long> entry : resUsage
- .getResourceUsageSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- resourceSecondsMap.put(entry.getKey(), value);
- }
- for (Map.Entry<String, Long> entry : attemptMetrics
- .getPreemptedResourceSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- preemptedSecondsMap.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ guaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap(),
+ opportunsiticResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ attemptMetrics.getPreemptedResourceSecondsMap(),
+ preemptedSecondsMap);
}
}
} finally {
@@ -1665,7 +1664,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
- numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
+ numAMContainerPreempted, guaranteedResourceSecondsMap,
+ preemptedSecondsMap, opportunsiticResourceSecondsMap);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 2bb7fd1..76272e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -28,18 +28,21 @@ public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
- private final Map<String, Long> resourceSecondsMap;
+ private final Map<String, Long> guaranteedResourceSecondsMap;
private final Map<String, Long> preemptedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
}
public Resource getResourcePreempted() {
@@ -54,17 +57,25 @@ public class RMAppMetrics {
return numAMContainersPreempted;
}
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
}
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
public long getPreemptedMemorySeconds() {
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
@@ -75,12 +86,15 @@ public class RMAppMetrics {
ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceSecondsMap() {
- return resourceSecondsMap;
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ return guaranteedResourceSecondsMap;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
index b858712..a75cc57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -27,30 +27,55 @@ import java.util.Map;
@Private
public class AggregateAppResourceUsage {
- private Map<String, Long> resourceSecondsMap = new HashMap<>();
+ private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
- public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap.putAll(resourceSecondsMap);
+ public AggregateAppResourceUsage(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap.putAll(
+ guaranteedResourceSecondsMap);
+ this.opportunisticResourceSecondsMap.putAll(
+ opportunisticResourceSecondsMap);
}
/**
- * @return the memorySeconds
+ * @return the guaranteed memory seconds
*/
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
/**
- * @return the vcoreSeconds
+ * @return the guaranteed vcore seconds
*/
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceUsageSecondsMap() {
- return resourceSecondsMap;
+ /**
+ * @return the opportunistic memory seconds
+ */
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+
+ /**
+ * @return the opportunistic vcore seconds
+ */
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 2d5c6a3..d716e7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -933,15 +933,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock.lock();
try {
ApplicationResourceUsageReport report =
- scheduler.getAppResourceUsageReport(this.getAppAttemptId());
+ scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId());
if (report == null) {
report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
}
AggregateAppResourceUsage resUsage =
this.attemptMetrics.getAggregateAppResourceUsage();
- report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
+ report.setGuaranteedResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap());
report.setPreemptedResourceSecondsMap(
this.attemptMetrics.getPreemptedResourceSecondsMap());
+ report.setOpportunisticResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap());
return report;
} finally {
this.readLock.unlock();
@@ -978,10 +981,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
- this.attemptMetrics
- .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage(
+ attemptState.getGuaranteedResourceSecondsMap());
this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
attemptState.getPreemptedResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage(
+ attemptState.getOpportunisticResourceSecondsMap());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1359,9 +1364,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
- stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
- getFinishTime(), resUsage.getResourceUsageSecondsMap(),
- this.attemptMetrics.getPreemptedResourceSecondsMap());
+ stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus,
+ exitStatus, getFinishTime(),
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ this.attemptMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 015cff7..192722b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -53,8 +53,11 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
- private Map<String, AtomicLong> resourceUsageMap = new HashMap<>();
+ private Map<String, AtomicLong> guaranteedResourceUsageMap
+ = new HashMap<>();
private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
+ private Map<String, AtomicLong> opportunisticResourceUsageMap
+ = new HashMap<>();
private RMContext rmContext;
private int[][] localityStatistics =
@@ -129,8 +132,10 @@ public class RMAppAttemptMetrics {
}
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
- Map<String, Long> resourcesUsed =
- convertAtomicLongMaptoLongMap(resourceUsageMap);
+ Map<String, Long> guaranteedResourcesUsed =
+ convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap);
+ Map<String, Long> opportunisticResourcesUsed =
+ convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap);
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
@@ -139,27 +144,25 @@ public class RMAppAttemptMetrics {
if (currentAttempt != null
&& currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport =
- rmContext.getScheduler().getAppResourceUsageReport(attemptId);
+ rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId);
if (appResUsageReport != null) {
- Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
- for (Map.Entry<String, Long> entry : tmp.entrySet()) {
- Long value = resourcesUsed.get(entry.getKey());
- if (value != null) {
- value += entry.getValue();
- } else {
- value = entry.getValue();
- }
- resourcesUsed.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getGuaranteedResourceSecondsMap(),
+ guaranteedResourcesUsed);
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getOpportunisticResourceSecondsMap(),
+ opportunisticResourcesUsed);
}
}
}
- return new AggregateAppResourceUsage(resourcesUsed);
+
+ return new AggregateAppResourceUsage(guaranteedResourcesUsed,
+ opportunisticResourcesUsed);
}
public void updateAggregateAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
- updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
+ updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
@@ -167,9 +170,14 @@ public class RMAppAttemptMetrics {
updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
}
- public void updateAggregateAppResourceUsage(
+ public void updateAggregateAppGuaranteedResourceUsage(
Map<String, Long> resourceSecondsMap) {
- updateUsageMap(resourceSecondsMap, resourceUsageMap);
+ updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap);
+ }
+
+ public void updateAggregateAppOpportunisticResourceUsage(
+ Map<String, Long> resourceSecondsMap) {
+ updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(
@@ -199,10 +207,9 @@ public class RMAppAttemptMetrics {
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
-
}
resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.set(entry.getValue());
+ resourceUsed.addAndGet(entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
new file mode 100644
index 0000000..c91dae8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import java.util.Map;
+
+/**
+ * The resource usage information of a <code>Container</code>.
+ */
+@Private
+public class ContainerResourceUsageReport {
+ private final Map<String, Long> guaranteedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
+
+ public ContainerResourceUsageReport(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index f3cbf63..4c43c5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
-import java.util.List;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -29,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -85,7 +82,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>,
ContainerState getContainerState();
ContainerReport createContainerReport();
-
+
+ ContainerResourceUsageReport getResourceUsageReport();
+
boolean isAMContainer();
ContainerRequest getContainerRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index e26689e..8f51d1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import java.util.Collections;
import java.util.EnumSet;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -39,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -176,7 +178,7 @@ public class RMContainerImpl implements RMContainer {
private NodeId reservedNode;
private SchedulerRequestKey reservedSchedulerKey;
private long creationTime;
- private long finishTime;
+ private long finishTime = -1L;
private ContainerStatus finishedStatus;
private boolean isAMContainer;
private ContainerRequest containerRequestForRecovery;
@@ -189,6 +191,10 @@ public class RMContainerImpl implements RMContainer {
private boolean isExternallyAllocated;
private SchedulerRequestKey allocatedSchedulerKey;
+ // This is updated whenever the container instance is updated in
+ // the case of container increase/decrease/promotion/demotion
+ private long lastContainerUpdateTime;
+
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
@@ -230,6 +236,7 @@ public class RMContainerImpl implements RMContainer {
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
+ this.lastContainerUpdateTime = creationTime;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
@@ -277,9 +284,87 @@ public class RMContainerImpl implements RMContainer {
}
public void setContainer(Container container) {
+ // containers are updated by resetting the underlying Container instance,
+ // always update RMContainer resource usage before the update.
+ updateAppAttemptMetrics(true);
this.container = container;
}
+ /**
+ * Collect the resource usage information of the current underlying Container
+ * instance and update its RMAppAttemptMetrics. This is called whenever the
+ * RMContainer is updated (by updating its underlying container) or finishes.
+ * @param containerUpdated if it is called upon container update event
+ */
+ private void updateAppAttemptMetrics(boolean containerUpdated) {
+ RMAppAttempt rmAttempt = rmContext.getRMApps()
+ .get(getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
+
+ if (rmAttempt != null) {
+ // collect resource usage information of the current Container instance
+ ContainerResourceUsageReport resourceUsage =
+ getCurrentContainerResourceUsage(containerUpdated);
+
+ // If this is a preempted container, update preemption metrics
+ if (finishedStatus != null &&
+ ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics()
+ .updatePreemptionInfo(container.getResource(), this);
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getOpportunisticResourceSecondsMap());
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppGuaranteedResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ }
+
+ /**
+ * Get resource usage of the current underlying Container instance.
+ * @param containerUpdated if this is called upon container update
+ * @return resource usage of the current <code>container</code>
+ */
+ private ContainerResourceUsageReport getCurrentContainerResourceUsage(
+ boolean containerUpdated) {
+ ContainerResourceUsageReport report;
+
+ // A container generates usage until it finishes which is indicated
+ // by a positive finish timestamp that is set once it finishes
+ final long currentTimeMillis =
+ finishTime < 0 ? System.currentTimeMillis() : finishTime;
+ final long usedSeconds = (currentTimeMillis -
+ lastContainerUpdateTime) / DateUtils.MILLIS_PER_SECOND;
+ Resource resource = container.getResource();
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ guaranteedResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(guaranteedResourceSeconds,
+ Collections.emptyMap());
+ } else {
+ Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ opportunisticResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(Collections.emptyMap(),
+ opportunisticResourceSeconds);
+ }
+
+ if (containerUpdated) {
+ lastContainerUpdateTime = currentTimeMillis;
+ }
+ return report;
+ }
+
@Override
public RMContainerState getState() {
this.readLock.lock();
@@ -683,7 +768,7 @@ public class RMContainerImpl implements RMContainer {
// Inform AppAttempt
// container.getContainer() can return null when a RMContainer is a
// reserved container
- updateAttemptMetrics(container);
+ container.updateAppAttemptMetrics(false);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
@@ -705,27 +790,6 @@ public class RMContainerImpl implements RMContainer {
}
}
-
- private static void updateAttemptMetrics(RMContainerImpl container) {
- Resource resource = container.getContainer().getResource();
- RMAppAttempt rmAttempt = container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId())
- .getCurrentAppAttempt();
-
- if (rmAttempt != null) {
- long usedMillis = container.finishTime - container.creationTime;
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregateAppResourceUsage(resource, usedMillis);
- // If this is a preempted container, update preemption metrics
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics()
- .updatePreemptionInfo(resource, container);
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
- }
- }
- }
}
private static final class KillTransition extends FinishedTransition {
@@ -764,6 +828,11 @@ public class RMContainerImpl implements RMContainer {
}
@Override
+ public ContainerResourceUsageReport getResourceUsageReport() {
+ return getCurrentContainerResourceUsage(false);
+ }
+
+ @Override
public String getNodeHttpAddress() {
try {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a755a4c..b78e93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -412,7 +412,7 @@ public abstract class AbstractYarnScheduler
}
@Override
- public ApplicationResourceUsageReport getAppResourceUsageReport(
+ public ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId) {
SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
if (attempt == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index dfb0e67..bfb160a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -60,12 +59,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -107,7 +106,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
- private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> lastGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ private Map<String, Long> lastOpportunisticResourceSecondsMap =
+ new HashMap<>();
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
@@ -1050,34 +1052,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// recently.
if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
> MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> aggregatedGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ Map<String, Long> aggregatedOpportunisticResourceSecondsMap =
+ new HashMap<>();
for (RMContainer rmContainer : this.liveContainers.values()) {
- long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
- Resource resource = rmContainer.getContainer().getResource();
- for (ResourceInformation entry : resource.getResources()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
- value += entry.getValue() * usedMillis
- / DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getName(), value);
- }
+ ContainerResourceUsageReport containerResourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ aggregatedGuaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getOpportunisticResourceSecondsMap(),
+ aggregatedOpportunisticResourceSecondsMap);
}
-
+ lastGuaranteedResourceSecondsMap =
+ aggregatedGuaranteedResourceSecondsMap;
+ lastOpportunisticResourceSecondsMap =
+ aggregatedOpportunisticResourceSecondsMap;
lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
- lastResourceSecondsMap = resourceSecondsMap;
}
- return new AggregateAppResourceUsage(lastResourceSecondsMap);
+ return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap,
+ lastOpportunisticResourceSecondsMap);
}
+ /**
+ * Get the resources that are actively being used by the app attempt.
+ * @return active resource usage
+ */
public ApplicationResourceUsageReport getResourceUsageReport() {
try {
writeLock.lock();
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
- Resource usedResourceClone = Resources.clone(
+ Resource guaranteedResourceUsedClone = Resources.clone(
attemptResourceUsage.getAllUsed());
+ Resource opportunisticResourceUsedClone = Resources.clone(
+ attemptOpportunisticResourceUsage.getAllUsed());
Resource reservedResourceClone = Resources.clone(
attemptResourceUsage.getReserved());
+ Resource neededResource = Resources.add(opportunisticResourceUsedClone,
+ Resources.add(guaranteedResourceUsedClone, reservedResourceClone));
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
@@ -1092,18 +1107,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
float queueCapacityPerc = queue.getQueueInfo(false, false)
.getCapacity();
if (queueCapacityPerc != 0) {
- queueUsagePerc = calc.divide(cluster, usedResourceClone,
+ queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone,
Resources.multiply(cluster, queueCapacityPerc)) * 100;
}
clusterUsagePerc =
- calc.divide(cluster, usedResourceClone, cluster) * 100;
+ calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100;
}
return ApplicationResourceUsageReport
.newInstance(liveContainers.size(), reservedContainers.size(),
- usedResourceClone, reservedResourceClone,
- Resources.add(usedResourceClone, reservedResourceClone),
- runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
- clusterUsagePerc, preemptedResourceSecondsMaps);
+ guaranteedResourceUsedClone, reservedResourceClone,
+ neededResource,
+ runningResourceUsage.getGuaranteedResourceUsageSecondsMap(),
+ queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps,
+ opportunisticResourceUsedClone,
+ runningResourceUsage.getOpportunisticResourceSecondsMap());
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 93ca7c2..a8ec8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -171,7 +171,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
*/
@LimitedPrivate("yarn")
@Evolving
- ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 21d6794..934c3a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1015,7 +1015,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
Resource effCap = ((AbstractCSQueue) getQueue())
.getEffectiveCapacity(getAppAMNodePartitionName());
float queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(), effCap) * 100;
+ report.getGuaranteedResourcesUsed(), effCap) * 100;
report.setQueueUsagePercentage(queueUsagePerc);
}
return report;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 8a89f78..d2e24bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
queue.getMetrics().releaseResources(
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
- this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedGuaranteedResource(containerResource);
+ if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
+ } else {
+ this.attemptOpportunisticResourceUsage.decUsed(containerResource);
+ }
+
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index ac88f86..14dab9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -95,6 +95,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
th(".runningcontainer", "Running Containers").
th(".allocatedCpu", "Allocated CPU VCores").
th(".allocatedMemory", "Allocated Memory MB").
+ th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU Vcores").
+ th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB").
th(".reservedCpu", "Reserved CPU VCores").
th(".reservedMemory", "Reserved Memory MB").
th(".progress", "Progress").
@@ -144,6 +148,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.valueOf(appInfo.getAllocatedVCores())).append("\",\"")
.append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
.valueOf(appInfo.getAllocatedMB())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"")
.append(appInfo.getReservedVCores() == -1 ? "N/A" : String
.valueOf(appInfo.getReservedVCores())).append("\",\"")
.append(appInfo.getReservedMB() == -1 ? "N/A" : String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 8553d8c..6cd10f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
.__("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
- .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
- StringHelper
- .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
+ .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getGuaranteedResourceSecondsMap()))
+ .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getOpportunisticResourceSecondsMap()))
.__("Aggregate Preempted Resource Allocation:",
appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
appMetrics.getPreemptedResourceSecondsMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index d0dccab..0510d62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -69,6 +69,10 @@ public class RMAppsBlock extends AppsBlock {
.th(".allocatedMemory", "Allocated Memory MB")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
+ .th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU VCores")
+ .th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
@@ -155,6 +159,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getReservedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getReservedMemoryMB()))
.append("\",\"")
+ .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedOpportunisticCpuVcores()))
+ .append("\",\"")
+ .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" :
+ String.valueOf(app.getAllocatedOpportunisticMemoryMB()))
+ .append("\",\"")
.append(queuePercent)
.append("\",\"")
.append(clusterPercent)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 0711b45..aeabbb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -94,6 +94,8 @@ public class AppInfo {
private String amRPCAddress;
private long allocatedMB;
private long allocatedVCores;
+ private long allocatedOpportunisticMB;
+ private long allocatedOpportunisticVCores;
private long reservedMB;
private long reservedVCores;
private int runningContainers;
@@ -102,6 +104,9 @@ public class AppInfo {
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
protected Map<String, Long> resourceSecondsMap;
+ private long opportunisticMemorySeconds;
+ private long opportunisticVcoreSeconds;
+ private Map<String, Long> opportunisticResourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@@ -199,10 +204,17 @@ public class AppInfo {
ApplicationResourceUsageReport resourceReport =
attempt.getApplicationResourceUsageReport();
if (resourceReport != null) {
- Resource usedResources = resourceReport.getUsedResources();
+ Resource guaranteedResourcesUsed =
+ resourceReport.getGuaranteedResourcesUsed();
+ Resource opportunisticResourceUsed =
+ resourceReport.getOpportunisticResourcesUsed();
Resource reservedResources = resourceReport.getReservedResources();
- allocatedMB = usedResources.getMemorySize();
- allocatedVCores = usedResources.getVirtualCores();
+ allocatedMB = guaranteedResourcesUsed.getMemorySize();
+ allocatedVCores = guaranteedResourcesUsed.getVirtualCores();
+ allocatedOpportunisticMB =
+ opportunisticResourceUsed.getMemorySize();
+ allocatedOpportunisticVCores =
+ opportunisticResourceUsed.getVirtualCores();
reservedMB = reservedResources.getMemorySize();
reservedVCores = reservedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
@@ -236,9 +248,13 @@ public class AppInfo {
numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
- memorySeconds = appMetrics.getMemorySeconds();
- vcoreSeconds = appMetrics.getVcoreSeconds();
- resourceSecondsMap = appMetrics.getResourceSecondsMap();
+ memorySeconds = appMetrics.getGuaranteedMemorySeconds();
+ vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds();
+ resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap();
+ opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds();
+ opportunisticResourceSecondsMap =
+ appMetrics.getOpportunisticResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
@@ -451,6 +467,13 @@ public class AppInfo {
return this.allocatedVCores;
}
+ public long getAllocatedOpportunisticMB() {
+ return this.allocatedOpportunisticMB;
+ }
+
+ public long getAllocatedOpportunisticVCores() {
+ return this.allocatedOpportunisticVCores;
+ }
public long getReservedMB() {
return this.reservedMB;
}
@@ -475,18 +498,30 @@ public class AppInfo {
return numAMContainerPreempted;
}
- public long getMemorySeconds() {
+ public long getGuaranteedMemorySeconds() {
return memorySeconds;
}
- public long getVcoreSeconds() {
+ public long getGuaranteedVcoreSeconds() {
return vcoreSeconds;
}
- public Map<String, Long> getResourceSecondsMap() {
+ public long getOpportunisticMemorySeconds() {
+ return opportunisticMemorySeconds;
+ }
+
+ public long getOpportunisticVcoreSeconds() {
+ return opportunisticVcoreSeconds;
+ }
+
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
return resourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@@ -579,6 +614,15 @@ public class AppInfo {
this.allocatedVCores = allocatedVCores;
}
+ public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) {
+ this.allocatedOpportunisticMB = allocatedOpportunisticMB;
+ }
+
+ public void setAllocatedOpportunisticVCores(
+ long allocatedOpportunisticVCores) {
+ this.allocatedOpportunisticVCores = allocatedOpportunisticVCores;
+ }
+
public void setReservedMB(long reservedMB) {
this.reservedMB = reservedMB;
}
@@ -591,12 +635,20 @@ public class AppInfo {
this.runningContainers = runningContainers;
}
- public void setMemorySeconds(long memorySeconds) {
- this.memorySeconds = memorySeconds;
+ public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) {
+ this.memorySeconds = guaranteedMemorySeconds;
+ }
+
+ public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) {
+ this.vcoreSeconds = guaranteedVcoreSeconds;
+ }
+
+ public void setOpportunisticMemorySeconds(long oppMemorySeconds) {
+ this.opportunisticMemorySeconds = oppMemorySeconds;
}
- public void setVcoreSeconds(long vcoreSeconds) {
- this.vcoreSeconds = vcoreSeconds;
+ public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) {
+ this.opportunisticVcoreSeconds = oppVcoreSeconds;
}
public void setAppId(String appId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 39a56a8..1721b1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -89,6 +89,7 @@ message ApplicationAttemptStateDataProto {
optional int64 preempted_vcore_seconds = 14;
repeated StringLongMapProto application_resource_usage_map = 15;
repeated StringLongMapProto preempted_resource_usage_map = 16;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 17;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 140c15f..6ad6006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -1031,12 +1031,20 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
- 10, 1, resourceSecondsMap, new HashMap<>());
+ 10, 1, guaranteedResourceSecondsMap,
+ new HashMap<>(), opportunisticResourceSecondsMap);
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
@@ -1047,16 +1055,19 @@ public class TestAppManager{
Assert.assertFalse(msg.contains("\r"));
String escaped = "\\n\\n\\r\\r";
- assertTrue(msg.contains("Multiline" + escaped +"AppName"));
- assertTrue(msg.contains("Multiline" + escaped +"UserName"));
- assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
- assertTrue(msg.contains("submitTime=1000"));
- assertTrue(msg.contains("memorySeconds=16384"));
- assertTrue(msg.contains("vcoreSeconds=64"));
- assertTrue(msg.contains("preemptedAMContainers=1"));
- assertTrue(msg.contains("preemptedNonAMContainers=10"));
- assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
- assertTrue(msg.contains("applicationType=MAPREDUCE"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+ Assert.assertTrue(msg.contains("submitTime=1000"));
+ Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+ Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+ Assert.assertTrue(
+ msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+ Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c..698ca90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
- -1, usageReport.getUsedResources().getMemorySize());
+ -1, usageReport.getGuaranteedResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemorySize());
+ Assert.assertEquals("Enemy should not see app used opportunistic resources",
+ -1, usageReport.getOpportunisticResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c95d31fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 1c50dd3..a66a91b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -370,8 +370,8 @@ public class TestClientRMService {
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
- Assert.assertEquals(10, usageReport.getMemorySeconds());
- Assert.assertEquals(3, usageReport.getVcoreSeconds());
+ Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds());
Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
@@ -1402,26 +1402,28 @@ public class TestClientRMService {
when(asContext.getMaxAppAttempts()).thenReturn(1);
when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
- RMAppImpl app =
- spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
+ ResourceRequest resourceRequest = BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ Resource.newInstance(1024, 1), 1);
+ RMAppImpl rmApp =
+ new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- Collections.singletonList(BuilderUtils.newResourceRequest(
- RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1))){
- @Override
- public ApplicationReport createAndGetApplicationReport(
- String clientUserName, boolean allowAccess) {
- ApplicationReport report = super.createAndGetApplicationReport(
- clientUserName, allowAccess);
- ApplicationResourceUsageReport usageReport =
- report.getApplicationResourceUsageReport();
- usageReport.setMemorySeconds(memorySeconds);
- usageReport.setVcoreSeconds(vcoreSeconds);
- report.setApplicationResourceUsageReport(usageReport);
- return report;
- }
- });
+ Collections.singletonList(resourceRequest)) {
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationReport report = super.createAndGetApplicationReport(
+ clientUserName, allowAccess);
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ usageReport.setGuaranteedMemorySeconds(memorySeconds);
+ usageReport.setGuaranteedVcoreSeconds(vcoreSeconds);
+ report.setApplicationResourceUsageReport(usageReport);
+ return report;
+ }
+ };
+ RMAppImpl app = spy(rmApp);
app.getAMResourceRequests().get(0)
.setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
@@ -1474,7 +1476,8 @@ public class TestClientRMService {
when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+ when(yarnScheduler.getAppActiveResourceUsageReport(attemptId))
+ .thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org