You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2013/03/27 15:42:41 UTC
svn commit: r1461614 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Author: tomwhite
Date: Wed Mar 27 14:42:41 2013
New Revision: 1461614
URL: http://svn.apache.org/r1461614
Log:
YARN-496. Fair scheduler configs are refreshed inconsistently in reinitialize. Contributed by Sandy Ryza
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1461614&r1=1461613&r2=1461614&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Mar 27 14:42:41 2013
@@ -146,6 +146,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-474. Fix CapacityScheduler to trigger application-activation when
am-resource-percent configuration is refreshed. (Zhijie Shen via vinodkv)
+ YARN-496. Fair scheduler configs are refreshed inconsistently in
+ reinitialize. (Sandy Ryza via tomwhite)
+
Release 2.0.4-alpha - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1461614&r1=1461613&r2=1461614&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Wed Mar 27 14:42:41 2013
@@ -921,25 +921,25 @@ public class FairScheduler implements Re
@Override
public synchronized void reinitialize(Configuration conf, RMContext rmContext)
throws IOException {
+ this.conf = new FairSchedulerConfiguration(conf);
+ minimumAllocation = this.conf.getMinimumMemoryAllocation();
+ maximumAllocation = this.conf.getMaximumMemoryAllocation();
+ userAsDefaultQueue = this.conf.getUserAsDefaultQueue();
+ nodeLocalityThreshold = this.conf.getLocalityThresholdNode();
+ rackLocalityThreshold = this.conf.getLocalityThresholdRack();
+ preemptionEnabled = this.conf.getPreemptionEnabled();
+ assignMultiple = this.conf.getAssignMultiple();
+ maxAssign = this.conf.getMaxAssign();
+ sizeBasedWeight = this.conf.getSizeBasedWeight();
+
if (!initialized) {
- this.conf = new FairSchedulerConfiguration(conf);
rootMetrics = QueueMetrics.forQueue("root", null, true, conf);
this.rmContext = rmContext;
this.eventLog = new FairSchedulerEventLog();
eventLog.init(this.conf);
- minimumAllocation = this.conf.getMinimumMemoryAllocation();
- maximumAllocation = this.conf.getMaximumMemoryAllocation();
- userAsDefaultQueue = this.conf.getUserAsDefaultQueue();
- nodeLocalityThreshold = this.conf.getLocalityThresholdNode();
- rackLocalityThreshold = this.conf.getLocalityThresholdRack();
- preemptionEnabled = this.conf.getPreemptionEnabled();
- assignMultiple = this.conf.getAssignMultiple();
- maxAssign = this.conf.getMaxAssign();
initialized = true;
- sizeBasedWeight = this.conf.getSizeBasedWeight();
-
try {
queueMgr.initialize();
} catch (Exception e) {
@@ -951,14 +951,8 @@ public class FairScheduler implements Re
updateThread.setDaemon(true);
updateThread.start();
} else {
- this.conf = new FairSchedulerConfiguration(conf);
- userAsDefaultQueue = this.conf.getUserAsDefaultQueue();
- nodeLocalityThreshold = this.conf.getLocalityThresholdNode();
- rackLocalityThreshold = this.conf.getLocalityThresholdRack();
- preemptionEnabled = this.conf.getPreemptionEnabled();
try {
queueMgr.reloadAllocs();
-
} catch (Exception e) {
throw new IOException("Failed to initialize FairScheduler", e);
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1461614&r1=1461613&r2=1461614&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Wed Mar 27 14:42:41 2013
@@ -37,7 +37,6 @@ public class FairSchedulerConfiguration
protected static final String USER_AS_DEFAULT_QUEUE = CONF_PREFIX + "user-as-default-queue";
protected static final boolean DEFAULT_USER_AS_DEFAULT_QUEUE = true;
- protected static final String LOCALITY_THRESHOLD = CONF_PREFIX + "locality.threshold";
protected static final float DEFAULT_LOCALITY_THRESHOLD = -1.0f;
/** Cluster threshold for node locality. */
@@ -89,10 +88,6 @@ public class FairSchedulerConfiguration
return getBoolean(USER_AS_DEFAULT_QUEUE, DEFAULT_USER_AS_DEFAULT_QUEUE);
}
- public float getLocalityThreshold() {
- return getFloat(LOCALITY_THRESHOLD, DEFAULT_LOCALITY_THRESHOLD);
- }
-
public float getLocalityThresholdNode() {
return getFloat(LOCALITY_THRESHOLD_NODE, DEFAULT_LOCALITY_THRESHOLD_NODE);
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1461614&r1=1461613&r2=1461614&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Wed Mar 27 14:42:41 2013
@@ -184,6 +184,26 @@ public class TestFairScheduler {
// TESTS
+ @Test(timeout=2000)
+ public void testLoadConfigurationOnInitialize() throws IOException {
+ Configuration conf = createConfiguration();
+ conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
+ conf.setInt(FairSchedulerConfiguration.MAX_ASSIGN, 3);
+ conf.setBoolean(FairSchedulerConfiguration.SIZE_BASED_WEIGHT, true);
+ conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_NODE, .5);
+ conf.setDouble(FairSchedulerConfiguration.LOCALITY_THRESHOLD_RACK, .7);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 1024);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 512);
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+ Assert.assertEquals(true, scheduler.assignMultiple);
+ Assert.assertEquals(3, scheduler.maxAssign);
+ Assert.assertEquals(true, scheduler.sizeBasedWeight);
+ Assert.assertEquals(.5, scheduler.nodeLocalityThreshold, .01);
+ Assert.assertEquals(.7, scheduler.rackLocalityThreshold, .01);
+ Assert.assertEquals(1024, scheduler.getMaximumResourceCapability().getMemory());
+ Assert.assertEquals(512, scheduler.getMinimumResourceCapability().getMemory());
+ }
+
@Test
public void testAggregateCapacityTracking() throws Exception {
// Add a node