You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/28 22:27:36 UTC
[09/24] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index a6b30b3..ad3413f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -23,17 +23,12 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.commons.lang3.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -43,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.log4j.Level;
@@ -84,13 +80,13 @@ public class TestContainerResourceUsage {
RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
- "Before app submittion, memory seconds should have been 0 but was "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() == 0);
+ "Before app submission, memory seconds should have been 0 but was "
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() == 0);
Assert.assertTrue(
"Before app submission, vcore seconds should have been 0 but was "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() == 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
@@ -105,7 +101,8 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
+ while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0
+ && cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
@@ -113,27 +110,35 @@ public class TestContainerResourceUsage {
rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
"While app is running, memory seconds should be >0 but is "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() > 0);
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() > 0);
Assert.assertTrue(
"While app is running, vcore seconds should be >0 but is "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() > 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() > 0);
MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ ru.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ ru.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ ru.getOpportunisticMemorySeconds(),
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ ru.getOpportunisticVcoreSeconds(),
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
}
- @Test (timeout = 120000)
+ @Test ()
public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
// Set max attempts to 1 so that when the first attempt fails, the app
// won't try to start a new one.
@@ -191,7 +196,7 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (app0.getRMAppMetrics().getMemorySeconds() <= 0
+ while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0
&& cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
@@ -216,19 +221,29 @@ public class TestContainerResourceUsage {
rm0.waitForState(nm, cId, RMContainerState.COMPLETED);
// Check that the container metrics match those from the app usage report.
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
for (RMContainer c : rmContainers) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics metricsBefore = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, metricsBefore.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, metricsBefore.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ metricsBefore.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ metricsBefore.getOpportunisticVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
MockRM rm1 = new MockRM(conf, memStore);
@@ -238,10 +253,22 @@ public class TestContainerResourceUsage {
// Compare container resource usage metrics from before and after restart.
RMAppMetrics metricsAfter = app0After.getRMAppMetrics();
- Assert.assertEquals("Vcore seconds were not the same after RM Restart",
- metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds());
- Assert.assertEquals("Memory seconds were not the same after RM Restart",
- metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds());
+ Assert.assertEquals(
+ "Guaranteed vcore seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedVcoreSeconds(),
+ metricsAfter.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(
+ "Guaranteed memory seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedMemorySeconds(),
+ metricsAfter.getGuaranteedMemorySeconds());
+ Assert.assertEquals(
+ "Opportunistic vcore seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticVcoreSeconds(),
+ metricsAfter.getOpportunisticVcoreSeconds());
+ Assert.assertEquals(
+ "Opportunistic memory seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticMemorySeconds(),
+ metricsAfter.getOpportunisticMemorySeconds());
rm0.stop();
rm0.close();
@@ -264,10 +291,8 @@ public class TestContainerResourceUsage {
MockRM rm = new MockRM(conf);
rm.start();
- RMApp app =
- rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
- null, "MAPREDUCE", false, keepRunningContainers);
+ RMApp app = rm.submitApp(200, "name", "user", new HashMap<>(), false,
+ "default", -1, null, "MAPREDUCE", false, keepRunningContainers);
MockNM nm =
new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
nm.registerNode();
@@ -275,18 +300,17 @@ public class TestContainerResourceUsage {
MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
int NUM_CONTAINERS = 1;
// allocate NUM_CONTAINERS containers
- am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
List<Container> containers =
- am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ am0.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
- containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers());
+ containers.addAll(am0.allocate(new ArrayList<>(),
+ new ArrayList<>()).getAllocatedContainers());
Thread.sleep(200);
}
@@ -294,26 +318,27 @@ public class TestContainerResourceUsage {
ContainerId containerId2 =
ContainerId.newContainerId(am0.getApplicationAttemptId(), 2);
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- containerId2.getContainerId(), ContainerState.RUNNING);
+ containerId2.getContainerId(), ContainerState.RUNNING);
rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
- Collection<RMContainer> rmContainers =
- rm.scheduler
- .getSchedulerAppInfo(am0.getApplicationAttemptId())
- .getLiveContainers();
+ Collection<RMContainer> rmContainers = rm.scheduler
+ .getSchedulerAppInfo(am0.getApplicationAttemptId())
+ .getLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId =
app.getCurrentAppAttempt().getMasterContainer().getId();
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- amContainerId.getContainerId(), ContainerState.COMPLETE);
+ amContainerId.getContainerId(), ContainerState.COMPLETE);
rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED);
rm.drainEvents();
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers) {
@@ -321,8 +346,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
if (c.getContainerId().equals(amContainerId)) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
} else {
// The remaining container should be RUNNING.
Assert.assertTrue("After first attempt failed, remaining container "
@@ -336,8 +363,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
}
@@ -355,14 +384,12 @@ public class TestContainerResourceUsage {
am1.registerAppAttempt();
rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// allocate NUM_CONTAINERS containers
- am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
- containers =
- am1.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ containers = am1.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
@@ -388,19 +415,26 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
- return;
}
private void waitforContainerCompletion(MockRM rm, MockNM nm,
@@ -419,16 +453,10 @@ public class TestContainerResourceUsage {
private AggregateAppResourceUsage calculateContainerResourceMetrics(
RMContainer rmContainer) {
- Resource resource = rmContainer.getContainer().getResource();
- long usedMillis =
- rmContainer.getFinishTime() - rmContainer.getCreationTime();
- long memorySeconds = resource.getMemorySize()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- long vcoreSeconds = resource.getVirtualCores()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- Map<String, Long> map = new HashMap<>();
- map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- return new AggregateAppResourceUsage(map);
+ ContainerResourceUsageReport resourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ return new AggregateAppResourceUsage(
+ resourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ resourceUsageReport.getOpportunisticResourceSecondsMap());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 342dab8..58be457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -209,7 +209,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
}
@Override
@@ -375,7 +375,7 @@ public abstract class MockAsm extends MockApps {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0,
- new HashMap<>());
+ new HashMap<>(), null, new HashMap<>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
index 5b0c34f..cdfd361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
@@ -479,7 +479,7 @@ public class TestCombinedSystemMetricsPublisher {
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(rmApp.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap,
- preemptedMap));
+ preemptedMap, new HashMap<>()));
when(rmApp.getApplicationTags()).thenReturn(
Collections.<String> emptySet());
ApplicationSubmissionContext appSubmissionContext =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index f2d82e9..a8f31e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -219,13 +219,25 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
- app.getRMAppMetrics().getMemorySeconds(),
+ app.getRMAppMetrics().getGuaranteedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getGuaranteedVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
- app.getRMAppMetrics().getVcoreSeconds(),
+ app.getRMAppMetrics().getOpportunisticMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getOpportunisticVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
app.getRMAppMetrics().getPreemptedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
@@ -517,7 +529,8 @@ public class TestSystemMetricsPublisher {
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics())
- .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
+ .thenReturn(new RMAppMetrics(null, 0, 0,
+ resourceMap, preemptedMap, new HashMap<>()));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 3b503e5..921d90c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -221,7 +221,7 @@ public class TestSystemMetricsPublisherForV2 {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(
- appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
+ appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0);
}
@Test(timeout = 10000)
@@ -365,14 +365,14 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus())
.thenReturn(FinalApplicationStatus.UNDEFINED);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
- resourceSecondsMap
+ guaranteedResourceSecondsMap
.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
- new HashMap<>()));
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
+ guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>()));
when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 3454d72..4f84567 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -202,7 +202,8 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -300,7 +301,8 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -377,7 +379,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -402,7 +405,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 11be3b1..12b6033 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -555,7 +555,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<>(),
+ new HashMap<>(0)));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -567,7 +568,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, new HashMap<>(), new HashMap<>());
+ finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -804,20 +805,33 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), 0L);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- 0L);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
+ Map<String, Long> guaranteedResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> opportunisticResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> preemptedResoureSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
- amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
+ amExitStatus, 0, guaranteedResourceSecondsMap,
+ preemptedResoureSecondsMap, opportunisticResourceSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 70887e0..6ee5e40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -243,7 +242,7 @@ public class TestRMAppTransitions {
ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
doReturn(null).when(resourceScheduler)
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
+ .getAppActiveResourceUsageReport(Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
@@ -1249,7 +1248,8 @@ public class TestRMAppTransitions {
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
- Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
+ Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT,
+ report.getApplicationResourceUsageReport());
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
Assert.assertTrue("bad proxy url for app",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 4a5c671..550e24d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -299,10 +299,12 @@ public class TestRMAppAttemptTransitions {
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L);
when(resourceScheduler
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
+ .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any()))
.thenReturn(appResUsgRpt);
spyRMContext = spy(rmContext);
Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
@@ -762,10 +764,12 @@ public class TestRMAppAttemptTransitions {
ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
- when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
- .thenReturn(appResUsgRpt);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L);
+ when(scheduler.getAppActiveResourceUsageReport(
+ any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
// start and finish the attempt
Container amContainer = allocateApplicationAttempt();
@@ -777,12 +781,14 @@ public class TestRMAppAttemptTransitions {
// expect usage stats to come from the scheduler report
ApplicationResourceUsageReport report =
applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(123456L, report.getMemorySeconds());
- Assert.assertEquals(55544L, report.getVcoreSeconds());
+ Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds());
+ Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds());
// finish app attempt and remove it from scheduler
- when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L);
sendAttemptUpdateSavedEvent(applicationAttempt);
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
@@ -793,8 +799,8 @@ public class TestRMAppAttemptTransitions {
when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
report = applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(223456, report.getMemorySeconds());
- Assert.assertEquals(75544, report.getVcoreSeconds());
+ Assert.assertEquals(223456, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index def0419..943a59a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -2838,7 +2838,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
Thread.sleep(1000);
AllocateResponse allocResponse = am1.schedule();
ApplicationResourceUsageReport report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
@@ -2851,7 +2851,7 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
allocResponse = am1.schedule();
}
report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index e239191..7a10d19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -223,23 +223,23 @@ public class TestCapacitySchedulerNodeLabelUpdate {
RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
ApplicationResourceUsageReport appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(1024, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(1, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
// request a container.
am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED);
appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(2048, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(2, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
LeafQueue queue =
(LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
.getQueue("a");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index f88ac8b..de97ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -66,7 +66,7 @@ public class TestAppPage {
RMAppMetrics appMetrics =
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 93377be..3f5926a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Token;
@@ -259,7 +258,7 @@ public class TestRMWebApp {
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getLaunchTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
- (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
+ null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index cea0088..87846c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -138,7 +138,7 @@ public class TestRMWebAppFairScheduler {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
- new HashMap<>(), new HashMap<>());
+ new HashMap<>(), new HashMap<>(), new HashMap<>());
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 15f94e1..8eb6023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1605,7 +1605,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
throws JSONException, Exception {
- int expectedNumberOfElements = 40 + (hasResourceReqs ? 2 : 0);
+ int expectedNumberOfElements = 45 + (hasResourceReqs ? 2 : 0);
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
index 83e0056..d034abb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
@@ -222,7 +222,7 @@ public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
}
private int getExpectedNumberOfElements(RMApp app) {
- int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
+ int expectedNumberOfElements = 45 + 2; // 2 -> resourceRequests
if (app.getApplicationSubmissionContext()
.getNodeLabelExpression() != null) {
expectedNumberOfElements++;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 40bdbd8..6f1ee3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -339,12 +339,22 @@ public final class RouterWebServiceUtil {
am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB());
am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores());
+ am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() +
+ uam.getAllocatedOpportunisticMB());
+ am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() +
+ uam.getAllocatedOpportunisticVCores());
am.setReservedMB(am.getReservedMB() + uam.getReservedMB());
am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB());
am.setRunningContainers(
am.getRunningContainers() + uam.getRunningContainers());
- am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds());
- am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds());
+ am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() +
+ uam.getGuaranteedMemorySeconds());
+ am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() +
+ uam.getGuaranteedVcoreSeconds());
+ am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() +
+ uam.getOpportunisticVcoreSeconds());
+ am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() +
+ uam.getOpportunisticMemorySeconds());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c045be51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index edf3804..1a5c42e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -249,8 +249,10 @@ public class TestRouterWebServiceUtil {
Assert.assertEquals(value * 3, app.getReservedMB());
Assert.assertEquals(value * 3, app.getReservedVCores());
Assert.assertEquals(value * 3, app.getRunningContainers());
- Assert.assertEquals(value * 3, app.getMemorySeconds());
- Assert.assertEquals(value * 3, app.getVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds());
Assert.assertEquals(3, app.getResourceRequests().size());
}
@@ -258,11 +260,15 @@ public class TestRouterWebServiceUtil {
am.getResourceRequests().add(new ResourceRequestInfo());
am.setAllocatedMB(value);
am.setAllocatedVCores(value);
+ am.setAllocatedOpportunisticVCores(value);
+ am.setAllocatedOpportunisticMB(value);
am.setReservedMB(value);
am.setReservedVCores(value);
am.setRunningContainers(value);
- am.setMemorySeconds(value);
- am.setVcoreSeconds(value);
+ am.setGuaranteedMemorySeconds(value);
+ am.setGuaranteedVcoreSeconds(value);
+ am.setOpportunisticMemorySeconds(value);
+ am.setOpportunisticVcoreSeconds(value);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org