You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/05/02 20:40:47 UTC
[16/51] [abbrv] hadoop git commit: YARN-6232. Update resource usage
and preempted resource calculations to take into account all resource types.
Contributed by Varun Vasudev.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 3f774e5..bbc35fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.util.StringHelper;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
@@ -106,15 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
._("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
- ._("Aggregate Resource Allocation:",
- String.format("%d MB-seconds, %d vcore-seconds",
- appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
- appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()))
+ ._("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
+ StringHelper
+ .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
._("Aggregate Preempted Resource Allocation:",
- String.format("%d MB-seconds, %d vcore-seconds",
- appMetrics == null ? "N/A" : appMetrics.getPreemptedMemorySeconds(),
- appMetrics == null ? "N/A" :
- appMetrics.getPreemptedVcoreSeconds()));
+ appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getPreemptedResourceSecondsMap()));
pdiv._();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 7fed3df..bff8612 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -101,6 +101,7 @@ public class AppInfo {
private long vcoreSeconds;
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
+ protected Map<String, Long> resourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@@ -109,6 +110,7 @@ public class AppInfo {
private int numAMContainerPreempted;
private long preemptedMemorySeconds;
private long preemptedVcoreSeconds;
+ protected Map<String, Long> preemptedResourceSecondsMap;
// list of resource requests
@XmlElement(name = "resourceRequests")
@@ -234,8 +236,10 @@ public class AppInfo {
appMetrics.getResourcePreempted().getVirtualCores();
memorySeconds = appMetrics.getMemorySeconds();
vcoreSeconds = appMetrics.getVcoreSeconds();
+ resourceSecondsMap = appMetrics.getResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
+ preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
ApplicationSubmissionContext appSubmissionContext =
app.getApplicationSubmissionContext();
unmanagedApplication = appSubmissionContext.getUnmanagedAM();
@@ -454,6 +458,22 @@ public class AppInfo {
return this.reservedVCores;
}
+ public long getPreemptedMB() {
+ return preemptedResourceMB;
+ }
+
+ public long getPreemptedVCores() {
+ return preemptedResourceVCores;
+ }
+
+ public int getNumNonAMContainersPreempted() {
+ return numNonAMContainerPreempted;
+ }
+
+ public int getNumAMContainersPreempted() {
+ return numAMContainerPreempted;
+ }
+
public long getMemorySeconds() {
return memorySeconds;
}
@@ -462,6 +482,10 @@ public class AppInfo {
return vcoreSeconds;
}
+ public Map<String, Long> getResourceSecondsMap() {
+ return resourceSecondsMap;
+ }
+
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@@ -470,6 +494,10 @@ public class AppInfo {
return preemptedVcoreSeconds;
}
+ public Map<String, Long> getPreemptedResourceSecondsMap() {
+ return preemptedResourceSecondsMap;
+ }
+
public List<ResourceRequestInfo> getResourceRequests() {
return this.resourceRequests;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
index 5083943..e13980a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
@@ -20,46 +20,68 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
@XmlRootElement
-@XmlAccessorType(XmlAccessType.FIELD)
+@XmlAccessorType(XmlAccessType.NONE)
public class ResourceInfo {
+
+ @XmlElement
long memory;
+ @XmlElement
int vCores;
-
+
+ private Resource resources;
+
public ResourceInfo() {
}
public ResourceInfo(Resource res) {
memory = res.getMemorySize();
vCores = res.getVirtualCores();
+ resources = Resources.clone(res);
}
public long getMemorySize() {
- return memory;
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
+ return resources.getMemorySize();
}
public int getvCores() {
- return vCores;
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
+ return resources.getVirtualCores();
}
-
+
@Override
public String toString() {
- return "<memory:" + memory + ", vCores:" + vCores + ">";
+ return resources.toString();
}
public void setMemory(int memory) {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
this.memory = memory;
+ resources.setMemorySize(memory);
}
public void setvCores(int vCores) {
+ if (resources == null) {
+ resources = Resource.newInstance(memory, vCores);
+ }
this.vCores = vCores;
+ resources.setVirtualCores(vCores);
}
public Resource getResource() {
- return Resource.newInstance(memory, vCores);
+ return Resource.newInstance(resources);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index cf93edd..887b854 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -73,7 +73,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
- return this.schedulingResourceTypes.toString();
+ return minAllocResource.getResource().getResources().keySet().toString();
}
public int getMaxClusterLevelAppPriority() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 247cd21..39a56a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -87,6 +87,8 @@ message ApplicationAttemptStateDataProto {
optional int64 finish_time = 12;
optional int64 preempted_memory_seconds = 13;
optional int64 preempted_vcore_seconds = 14;
+ repeated StringLongMapProto application_resource_usage_map = 15;
+ repeated StringLongMapProto preempted_resource_usage_map = 16;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index cd65879..1864caa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -35,6 +35,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -826,9 +828,12 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
+ Map<String, Long> resourceSecondsMap = new HashMap<>();
+ resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
+ resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
- 10, 1, 16384, 64, 0, 0);
+ 10, 1, resourceSecondsMap, new HashMap<String, Long>());
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index 11fe056..3508ab4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -424,6 +426,9 @@ public class TestContainerResourceUsage {
* usedMillis / DateUtils.MILLIS_PER_SECOND;
long vcoreSeconds = resource.getVirtualCores()
* usedMillis / DateUtils.MILLIS_PER_SECOND;
- return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
+ Map<String, Long> map = new HashMap<>();
+ map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
+ map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
+ return new AggregateAppResourceUsage(map);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index f826631..cdb3ba1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -189,7 +190,8 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
- return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0);
+ return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<String, Long>(),
+ new HashMap<String, Long>());
}
@Override
@@ -337,8 +339,9 @@ public abstract class MockAsm extends MockApps {
public ApplicationReport createAndGetApplicationReport(
String clientUserName, boolean allowAccess) {
ApplicationResourceUsageReport usageReport =
- ApplicationResourceUsageReport.newInstance(0, 0, null, null, null,
- 0, 0, 0, 0, 0, 0);
+ ApplicationResourceUsageReport
+ .newInstance(0, 0, null, null, null, new HashMap<String, Long>(), 0, 0,
+ new HashMap<String, Long>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 5321916..36e35d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -24,6 +24,7 @@ import static org.mockito.Mockito.when;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
@@ -40,6 +41,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -505,9 +507,16 @@ public class TestSystemMetricsPublisher {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus()).thenReturn(
FinalApplicationStatus.UNDEFINED);
- when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
- Integer.MAX_VALUE, Long.MAX_VALUE));
+ Map<String, Long> resourceMap = new HashMap<>();
+ resourceMap
+ .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
+ resourceMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
+ Map<String, Long> preemptedMap = new HashMap<>();
+ preemptedMap
+ .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
+ preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
+ when(app.getRMAppMetrics())
+ .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index ad71f6e..3aeafc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -29,6 +29,8 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -46,6 +48,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@@ -358,15 +361,20 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getDiagnostics()).thenReturn(
new StringBuilder("test diagnostics info"));
RMAppAttempt appAttempt = mock(RMAppAttempt.class);
- when(appAttempt.getAppAttemptId()).thenReturn(
- ApplicationAttemptId.newInstance(appId, 1));
+ when(appAttempt.getAppAttemptId())
+ .thenReturn(ApplicationAttemptId.newInstance(appId, 1));
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
- when(app.getFinalApplicationStatus()).thenReturn(
- FinalApplicationStatus.UNDEFINED);
+ when(app.getFinalApplicationStatus())
+ .thenReturn(FinalApplicationStatus.UNDEFINED);
+ Map<String, Long> resourceSecondsMap = new HashMap<>();
+ resourceSecondsMap
+ .put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
+ resourceSecondsMap
+ .put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, Integer.MAX_VALUE,
- Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE));
- when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
+ new HashMap<String, Long>()));
+ when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
when(appSubmissionContext.getPriority())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index fdac89c..a1339f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -194,7 +194,7 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(0, 0));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<String, Long>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -292,7 +292,7 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(0,0));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<String, Long>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -369,7 +369,7 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), 0, 0, 0, 0);
+ oldAttemptState.getFinishTime(), new HashMap<String, Long>(), new HashMap<String, Long>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -393,7 +393,7 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), 0, 0, 0, 0);
+ oldAttemptState.getFinishTime(), new HashMap<String, Long>(), new HashMap<String, Long>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 9126df8..729b6ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -33,12 +33,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.conf.HAUtil;
@@ -547,7 +542,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(0,0));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<String, Long>()));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -559,7 +554,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, 0, 0, 0, 0);
+ finishTime, new HashMap<String, Long>(), new HashMap<String, Long>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -796,10 +791,20 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
+ Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
+ resourceSecondsMap
+ .put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ resourceSecondsMap
+ .put(ResourceInformation.VCORES.getName(), 0L);
+ preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
+ 0L);
+ preemptedResoureSecondsMap
+ .put(ResourceInformation.VCORES.getName(), 0L);
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
- amExitStatus, 0, 0, 0, 0, 0);
+ amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index d9ed073..e67e1cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -22,6 +22,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.util.HashMap;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
@@ -62,9 +63,10 @@ public class TestAppPage {
when(app.getStartTime()).thenReturn(0L);
when(app.getFinishTime()).thenReturn(0L);
when(app.createApplicationState()).thenReturn(YarnApplicationState.FAILED);
-
- RMAppMetrics appMetrics = new RMAppMetrics(
- Resource.newInstance(0, 0), 0, 0, 0, 0, 0, 0);
+
+ RMAppMetrics appMetrics =
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<String, Long>(),
+ new HashMap<String, Long>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/031fea40/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index 8c00b39..6bd78c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -51,6 +51,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.concurrent.ConcurrentMap;
@@ -136,8 +137,8 @@ public class TestRMWebAppFairScheduler {
MockRMApp app = new MockRMApp(i, i, state) {
@Override
public RMAppMetrics getRMAppMetrics() {
- return new RMAppMetrics(Resource.newInstance(0, 0),
- 0, 0, 0, 0, 0, 0);
+ return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
+ new HashMap<String, Long>(), new HashMap<String, Long>());
}
@Override
public YarnApplicationState createApplicationState() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org