You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/01/19 06:04:47 UTC
hadoop git commit: YARN-4565. Fix a bug that leads to AM resource
limit not hornored when sizeBasedWeight enabled for FairOrderingPolicy.
Contributed by Wangda Tan
Repository: hadoop
Updated Branches:
refs/heads/trunk 92c5f565f -> edc43a909
YARN-4565. Fix a bug that leads to AM resource limit not hornored when sizeBasedWeight enabled for FairOrderingPolicy. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edc43a90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edc43a90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edc43a90
Branch: refs/heads/trunk
Commit: edc43a9097530fd469dee47d4fefd091818331e5
Parents: 92c5f56
Author: Jian He <ji...@apache.org>
Authored: Mon Jan 18 18:28:05 2016 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Jan 18 21:04:36 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../scheduler/SchedulerApplicationAttempt.java | 2 +-
.../scheduler/capacity/CapacityScheduler.java | 3 +-
.../CapacitySchedulerConfiguration.java | 13 +++++
.../policy/TestFairOrderingPolicy.java | 54 ++++++++++++++++++++
5 files changed, 73 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c1430d7..29ee57b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1280,6 +1280,9 @@ Release 2.8.0 - UNRELEASED
YARN-4502. Fix two AM containers get allocated when AM restart.
(Vinod Kumar Vavilapalli via wangda)
+ YARN-4565. Fix a bug that leads to AM resource limit not hornored when
+ sizeBasedWeight enabled for FairOrderingPolicy. (wtan via jianhe)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index d91c79e..ca05fe9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -863,7 +863,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
@Override
- public synchronized ResourceUsage getSchedulingResourceUsage() {
+ public ResourceUsage getSchedulingResourceUsage() {
return attemptResourceUsage;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index f4129ab..e773384 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -978,7 +978,8 @@ public class CapacityScheduler extends
clusterResource, getMinimumResourceCapability());
}
- if (updateDemandForQueue != null) {
+ if (updateDemandForQueue != null && !application
+ .isWaitingForAMContainer()) {
updateDemandForQueue.getOrderingPolicy().demandUpdated(application);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 1e62b44..2e8fd24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -30,6 +30,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.StringTokenizer;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -959,4 +960,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
DEFAULT_CONFIGURATION_APPLICATION_PRIORITY);
return defaultPriority;
}
+
+ @VisibleForTesting
+ public void setOrderingPolicy(String queue, String policy) {
+ set(getQueuePrefix(queue) + ORDERING_POLICY, policy);
+ }
+
+ @VisibleForTesting
+ public void setOrderingPolicyParameter(String queue,
+ String parameterKey, String parameterValue) {
+ set(getQueuePrefix(queue) + ORDERING_POLICY + "."
+ + parameterKey, parameterValue);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edc43a90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
index ffb9d93..98cfdab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/policy/TestFairOrderingPolicy.java
@@ -20,6 +20,18 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy;
import java.util.*;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.junit.Assert;
import org.junit.Test;
@@ -138,6 +150,48 @@ public class TestFairOrderingPolicy {
checkIds(schedOrder.getPreemptionIterator(), new String[]{"2", "1", "3"});
}
+ @Test
+ public void testSizeBasedWeightNotAffectAppActivation() throws Exception {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+
+ // Define top-level queues
+ String queuePath = CapacitySchedulerConfiguration.ROOT + ".default";
+ csConf.setOrderingPolicy(queuePath, CapacitySchedulerConfiguration.FAIR_ORDERING_POLICY);
+ csConf.setOrderingPolicyParameter(queuePath,
+ FairOrderingPolicy.ENABLE_SIZE_BASED_WEIGHT, "true");
+ csConf.setMaximumApplicationMasterResourcePerQueuePercent(queuePath, 0.1f);
+
+ // inject node label manager
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // Get LeafQueue
+ LeafQueue lq = (LeafQueue) cs.getQueue("default");
+ OrderingPolicy<FiCaSchedulerApp> policy = lq.getOrderingPolicy();
+ Assert.assertTrue(policy instanceof FairOrderingPolicy);
+ Assert.assertTrue(((FairOrderingPolicy<FiCaSchedulerApp>)policy).getSizeBasedWeight());
+
+ rm.registerNode("h1:1234", 10 * GB);
+
+ // Submit 4 apps
+ rm.submitApp(1 * GB, "app", "user", null, "default");
+ rm.submitApp(1 * GB, "app", "user", null, "default");
+ rm.submitApp(1 * GB, "app", "user", null, "default");
+ rm.submitApp(1 * GB, "app", "user", null, "default");
+
+ Assert.assertEquals(1, lq.getNumActiveApplications());
+ Assert.assertEquals(3, lq.getNumPendingApplications());
+
+ // Try allocate once, #active-apps and #pending-apps should be still correct
+ cs.handle(new NodeUpdateSchedulerEvent(
+ rm.getRMContext().getRMNodes().get(NodeId.newInstance("h1", 1234))));
+ Assert.assertEquals(1, lq.getNumActiveApplications());
+ Assert.assertEquals(3, lq.getNumPendingApplications());
+ }
+
public void checkIds(Iterator<MockSchedulableEntity> si,
String[] ids) {
for (int i = 0;i < ids.length;i++) {