You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/04/17 18:18:08 UTC
[26/45] hadoop git commit: YARN-6439. Fix ReservationSystem creation
of default ReservationQueue. (Carlo Curino via wangda)
YARN-6439. Fix ReservationSystem creation of default ReservationQueue. (Carlo Curino via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b286228
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b286228
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b286228
Branch: refs/heads/HDFS-10467
Commit: 8b2862283767b031d1e17e3db812fb9cdad552ad
Parents: 7d33e51
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Apr 11 14:56:18 2017 -0700
Committer: Inigo <in...@apache.org>
Committed: Mon Apr 17 11:17:01 2017 -0700
----------------------------------------------------------------------
.../capacity/CapacitySchedulerQueueManager.java | 20 ++++++++++++++++++++
.../scheduler/capacity/PlanQueue.java | 4 ++--
.../TestCapacitySchedulerPlanFollower.java | 7 +++++++
.../TestFairSchedulerPlanFollower.java | 4 ++++
.../TestSchedulerPlanFollowerBase.java | 9 ++++++++-
.../TestCapacitySchedulerDynamicBehavior.java | 11 +++++++++++
6 files changed, 52 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index c92c343..be6243d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -41,10 +41,13 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.Permission;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.security.AppPriorityACLsManager;
import com.google.common.annotations.VisibleForTesting;
@@ -220,6 +223,23 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
queue =
new PlanQueue(csContext, queueName, parent,
oldQueues.get(queueName));
+
+ //initializing the "internal" default queue, for SLS compatibility
+ String defReservationId =
+ queueName + ReservationConstants.DEFAULT_QUEUE_SUFFIX;
+
+ List<CSQueue> childQueues = new ArrayList<>();
+ ReservationQueue resQueue = new ReservationQueue(csContext,
+ defReservationId, (PlanQueue) queue);
+ try {
+ resQueue.setEntitlement(new QueueEntitlement(1.0f, 1.0f));
+ } catch (SchedulerDynamicEditException e) {
+ throw new IllegalStateException(e);
+ }
+ childQueues.add(resQueue);
+ ((PlanQueue) queue).setChildQueues(childQueues);
+ queues.put(defReservationId, resQueue);
+
} else {
queue =
new LeafQueue(csContext, queueName, parent,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index a391f25..882262f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -93,10 +93,10 @@ public class PlanQueue extends ParentQueue {
PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
- if (newlyParsedParentQueue.getChildQueues().size() > 0) {
+ if (newlyParsedParentQueue.getChildQueues().size() != 1) {
throw new IOException(
"Reservable Queue should not have sub-queues in the"
- + "configuration");
+ + "configuration expect the default reservation queue");
}
// Set new configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
index bccd5c3..4320d3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -154,6 +154,13 @@ public class TestCapacitySchedulerPlanFollower extends
}
@Override
+ protected void checkDefaultQueueBeforePlanFollowerRun(){
+ Queue defQ = getDefaultQueue();
+ Assert.assertEquals(0, getNumberOfApplications(defQ));
+ Assert.assertNotNull(defQ);
+ }
+
+ @Override
protected Queue getDefaultQueue() {
return cs.getQueue("dedicated" + ReservationConstants.DEFAULT_QUEUE_SUFFIX);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
index 1b4e2f8..9561234 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestFairSchedulerPlanFollower.java
@@ -132,6 +132,10 @@ public class TestFairSchedulerPlanFollower extends
}
@Override
+ protected void checkDefaultQueueBeforePlanFollowerRun() {
+ Assert.assertNull(getDefaultQueue());
+ }
+ @Override
protected void verifyCapacity(Queue defQ) {
assertTrue(((FSQueue) defQ).getWeights().getWeight(ResourceType.MEMORY) > 0.9);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
index b604799..c6cebd9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestSchedulerPlanFollowerBase.java
@@ -89,6 +89,11 @@ public abstract class TestSchedulerPlanFollowerBase {
"dedicated", 10, 10 + f2.length, ReservationSystemTestUtil
.generateAllocation(10L, 1L, f2), res, minAlloc), false));
+
+ // default reseration queue should exist before run of PlanFollower AND have
+ // no apps
+ checkDefaultQueueBeforePlanFollowerRun();
+
AbstractSchedulerPlanFollower planFollower = createPlanFollower();
when(mClock.getTime()).thenReturn(0L);
@@ -108,8 +113,8 @@ public abstract class TestSchedulerPlanFollowerBase {
new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
scheduler.handle(appAttemptAddedEvent);
- // initial default reservation queue should have no apps
+ // initial default reservation queue should have no apps after first run
Queue defQ = getDefaultQueue();
Assert.assertEquals(0, getNumberOfApplications(defQ));
@@ -179,6 +184,8 @@ public abstract class TestSchedulerPlanFollowerBase {
verifyCapacity(defQ);
}
+ protected abstract void checkDefaultQueueBeforePlanFollowerRun();
+
protected abstract Queue getReservationQueue(String reservationId);
protected abstract void verifyCapacity(Queue defQ);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b286228/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
index ce3382f..483ba1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerDynamicBehavior.java
@@ -76,6 +76,12 @@ public class TestCapacitySchedulerDynamicBehavior {
public void testRefreshQueuesWithReservations() throws Exception {
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ //set default queue capacity to zero
+ ((ReservationQueue) cs
+ .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
+ .setEntitlement(
+ new QueueEntitlement(0f, 1f));
+
// Test add one reservation dynamically and manually modify capacity
ReservationQueue a1 =
new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
@@ -120,6 +126,11 @@ public class TestCapacitySchedulerDynamicBehavior {
ReservationQueue a1 =
new ReservationQueue(cs, "a1", (PlanQueue) cs.getQueue("a"));
cs.addQueue(a1);
+ //set default queue capacity to zero
+ ((ReservationQueue) cs
+ .getQueue("a" + ReservationConstants.DEFAULT_QUEUE_SUFFIX))
+ .setEntitlement(
+ new QueueEntitlement(0f, 1f));
a1.setEntitlement(new QueueEntitlement(A1_CAPACITY / 100, 1f));
// Test add another reservation queue and use setEntitlement to modify
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org