You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by eb...@apache.org on 2020/08/05 23:15:00 UTC
[hadoop] branch branch-3.1 updated: YARN-4575.
ApplicationResourceUsageReport should return ALL reserved resource.
Contributed by Bibin Chundatt and Eric Payne.
This is an automated email from the ASF dual-hosted git repository.
ebadger pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 647be0c YARN-4575. ApplicationResourceUsageReport should return ALL reserved resource. Contributed by Bibin Chundatt and Eric Payne.
647be0c is described below
commit 647be0c0f69887afd7ad63aa1dfa6373a4b65ae6
Author: Eric Badger <eb...@verizonmedia.com>
AuthorDate: Wed Aug 5 23:14:27 2020 +0000
YARN-4575. ApplicationResourceUsageReport should return ALL reserved resource.
Contributed by Bibin Chundatt and Eric Payne.
---
.../scheduler/SchedulerApplicationAttempt.java | 4 +--
.../scheduler/common/fica/FiCaSchedulerApp.java | 3 +-
.../scheduler/TestSchedulerApplicationAttempt.java | 39 ++++++++++++++++++++++
3 files changed, 43 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 9102f36..d3453d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -1113,8 +1113,8 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
getRunningAggregateAppResourceUsage();
Resource usedResourceClone = Resources.clone(
attemptResourceUsage.getAllUsed());
- Resource reservedResourceClone = Resources.clone(
- attemptResourceUsage.getReserved());
+ Resource reservedResourceClone =
+ Resources.clone(attemptResourceUsage.getAllReserved());
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 589e9a1..0e52339 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -716,7 +716,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
+ " on node " + node + ", currently has "
+ reservedContainers.size()
+ " at priority " + schedulerKey.getPriority()
- + "; currentReservation " + this.attemptResourceUsage.getReserved()
+ + "; currentReservation "
+ + this.attemptResourceUsage.getReserved(node.getPartition())
+ " on node-label=" + node.getPartition());
return true;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index fd3e4ae..7d9fcb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils.toSchedulerKey;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Test;
@@ -312,6 +314,43 @@ public class TestSchedulerApplicationAttempt {
}
@Test
+ public void testAllResourceUsage() throws Exception {
+ FifoScheduler scheduler = mock(FifoScheduler.class);
+ when(scheduler.getClusterResource()).thenReturn(Resource.newInstance(0, 0));
+ when(scheduler.getResourceCalculator())
+ .thenReturn(new DefaultResourceCalculator());
+
+ ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ when(rmContext.getYarnConfiguration()).thenReturn(conf);
+
+ final String user = "user1";
+ Queue queue = createQueue("test", null);
+ SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
+ user, queue, queue.getAbstractUsersManager(), rmContext);
+
+ // Resource request
+ Resource requestedResource = Resource.newInstance(1536, 2);
+ app.attemptResourceUsage.incUsed("X", requestedResource);
+ app.attemptResourceUsage.incUsed("Y", requestedResource);
+ Resource r2 = Resource.newInstance(1024, 1);
+ app.attemptResourceUsage.incReserved("X", r2);
+ app.attemptResourceUsage.incReserved("Y", r2);
+
+ assertTrue("getUsedResources expected " + Resource.newInstance(3072, 4)
+ + " but was " + app.getResourceUsageReport().getUsedResources(),
+ Resources.equals(Resource.newInstance(3072, 4),
+ app.getResourceUsageReport().getUsedResources()));
+ assertTrue("getReservedResources expected " + Resource.newInstance(2048, 2)
+ + " but was "
+ + app.getResourceUsageReport().getReservedResources(),
+ Resources.equals(Resource.newInstance(2048, 2),
+ app.getResourceUsageReport().getReservedResources()));
+ }
+
+ @Test
public void testSchedulingOpportunityOverflow() throws Exception {
ApplicationAttemptId attemptId = createAppAttemptId(0, 0);
Queue queue = createQueue("test", null);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org