You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2012/11/10 01:49:25 UTC
svn commit: r1407706 - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache...
Author: szetszwo
Date: Sat Nov 10 00:49:15 2012
New Revision: 1407706
URL: http://svn.apache.org/viewvc?rev=1407706&view=rev
Log:
Merge r1406415 through r1407703 from trunk.
Added:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error
- copied unchanged from r1407703, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/mock-container-executer-with-error
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
- copied unchanged from r1407703, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
Removed:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewJobWeightBooster.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CLIMiniCluster.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/EncryptedShuffle.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/Federation.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailabilityWithNFS.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HDFSHighAvailabilityWithQJM.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebHDFS.apt.vm
Modified:
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/CHANGES.txt Sat Nov 10 00:49:15 2012
@@ -69,6 +69,8 @@ Release 2.0.3-alpha - Unreleased
YARN-136. Make ClientToAMTokenSecretManager part of RMContext (Vinod Kumar
Vavilapalli via sseth)
+ YARN-183. Clean up fair scheduler code. (Sandy Ryza via tomwhite)
+
OPTIMIZATIONS
BUG FIXES
@@ -96,6 +98,9 @@ Release 2.0.3-alpha - Unreleased
YARN-181. Fixed eclipse settings broken by capacity-scheduler.xml move via
YARN-140. (Siddharth Seth via vinodkv)
+ YARN-169. Update log4j.appender.EventCounter to use
+ org.apache.hadoop.log.metrics.EventCounter (Anthony Rojas via tomwhite)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
@@ -166,6 +171,9 @@ Release 0.23.5 - UNRELEASED
YARN-32. Fix TestApplicationTokens to not depend on test order and thus pass
on JDK7. (vinodkv)
+ YARN-186. Coverage fixing LinuxContainerExecutor (Aleksey Gorshkov via
+ bobby)
+
OPTIMIZATIONS
BUG FIXES
@@ -202,6 +210,12 @@ Release 0.23.5 - UNRELEASED
YARN-202. Log Aggregation generates a storm of fsync() for namenode
(Kihwal Lee via bobby)
+ YARN-201. Fix CapacityScheduler to be less conservative for starved
+ off-switch requests. (jlowe via acmurthy)
+
+ YARN-206. TestApplicationCleanup.testContainerCleanup occasionally fails.
+ (jlowe via jeagles)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/resources/container-log4j.properties Sat Nov 10 00:49:15 2012
@@ -26,4 +26,4 @@ log4j.appender.CLA.layout.ConversionPatt
# Event Counter Appender
# Sends counts of logging messages at different severity levels to Hadoop Metrics.
#
-log4j.appender.EventCounter=org.apache.hadoop.metrics.jvm.EventCounter
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java Sat Nov 10 00:49:15 2012
@@ -26,11 +26,13 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.LineNumberReader;
+import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
+import junit.framework.Assert;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -46,7 +48,6 @@ import org.junit.Test;
public class TestLinuxContainerExecutorWithMocks {
- @SuppressWarnings("unused")
private static final Log LOG = LogFactory
.getLog(TestLinuxContainerExecutorWithMocks.class);
@@ -54,6 +55,7 @@ public class TestLinuxContainerExecutorW
private final File mockParamFile = new File("./params.txt");
private LocalDirsHandlerService dirsHandler;
+
private void deleteMockParamFile() {
if(mockParamFile.exists()) {
mockParamFile.delete();
@@ -126,8 +128,102 @@ public class TestLinuxContainerExecutorW
StringUtils.join(",", dirsHandler.getLocalDirs()),
StringUtils.join(",", dirsHandler.getLogDirs())),
readMockParams());
+
+ }
+
+
+ @Test
+ public void testStartLocalizer() throws IOException {
+
+
+ InetSocketAddress address = InetSocketAddress.createUnresolved("localhost", 8040);
+ Path nmPrivateCTokensPath= new Path("file:///bin/nmPrivateCTokensPath");
+
+ try {
+ mockExec.startLocalizer(nmPrivateCTokensPath, address, "test", "application_0", "12345", dirsHandler.getLocalDirs(), dirsHandler.getLogDirs());
+ List<String> result=readMockParams();
+ Assert.assertEquals(result.size(), 16);
+ Assert.assertEquals(result.get(0), "test");
+ Assert.assertEquals(result.get(1), "0" );
+ Assert.assertEquals(result.get(2),"application_0" );
+ Assert.assertEquals(result.get(3), "/bin/nmPrivateCTokensPath");
+ Assert.assertEquals(result.get(7), "-classpath" );
+ Assert.assertEquals(result.get(10),"org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer" );
+ Assert.assertEquals(result.get(11), "test");
+ Assert.assertEquals(result.get(12), "application_0");
+ Assert.assertEquals(result.get(13),"12345" );
+ Assert.assertEquals(result.get(14),"localhost" );
+ Assert.assertEquals(result.get(15),"8040" );
+
+ } catch (InterruptedException e) {
+ LOG.error("Error:"+e.getMessage(),e);
+ Assert.fail();
+ }
+ }
+
+
+ @Test
+ public void testContainerLaunchError() throws IOException {
+
+ // reinitialize executer
+ File f = new File("./src/test/resources/mock-container-executer-with-error");
+ if (!f.canExecute()) {
+ f.setExecutable(true);
+ }
+ String executorPath = f.getAbsolutePath();
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
+ conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
+
+ mockExec = new LinuxContainerExecutor();
+ dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+ mockExec.setConf(conf);
+
+ String appSubmitter = "nobody";
+ String cmd = String
+ .valueOf(LinuxContainerExecutor.Commands.LAUNCH_CONTAINER.getValue());
+ String appId = "APP_ID";
+ String containerId = "CONTAINER_ID";
+ Container container = mock(Container.class);
+ ContainerId cId = mock(ContainerId.class);
+ ContainerLaunchContext context = mock(ContainerLaunchContext.class);
+ HashMap<String, String> env = new HashMap<String, String>();
+
+ when(container.getContainerID()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(context);
+
+ when(cId.toString()).thenReturn(containerId);
+
+ when(context.getEnvironment()).thenReturn(env);
+
+ Path scriptPath = new Path("file:///bin/echo");
+ Path tokensPath = new Path("file:///dev/null");
+ Path workDir = new Path("/tmp");
+ Path pidFile = new Path(workDir, "pid.txt");
+
+ mockExec.activateContainer(cId, pidFile);
+ int ret = mockExec.launchContainer(container, scriptPath, tokensPath,
+ appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
+ dirsHandler.getLogDirs());
+ Assert.assertNotSame(0, ret);
+ assertEquals(Arrays.asList(appSubmitter, cmd, appId, containerId,
+ workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(),
+ StringUtils.join(",", dirsHandler.getLocalDirs()),
+ StringUtils.join(",", dirsHandler.getLogDirs())), readMockParams());
+
}
+
+ @Test
+ public void testInit() throws Exception {
+ mockExec.init();
+ assertEquals(Arrays.asList("--checksetup"), readMockParams());
+
+ }
+
+
@Test
public void testContainerKill() throws IOException {
String appSubmitter = "nobody";
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Sat Nov 10 00:49:15 2012
@@ -833,8 +833,12 @@ public class LeafQueue implements CSQueu
// Note: Update headroom to account for current allocation too...
allocateResource(clusterResource, application, assigned);
- // Reset scheduling opportunities
- application.resetSchedulingOpportunities(priority);
+ // Don't reset scheduling opportunities for non-local assignments
+ // otherwise the app will be delayed for each non-local assignment.
+ // This helps apps with many off-cluster requests schedule faster.
+ if (assignment.getType() != NodeType.OFF_SWITCH) {
+ application.resetSchedulingOpportunities(priority);
+ }
// Done
return assignment;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Sat Nov 10 00:49:15 2012
@@ -101,7 +101,7 @@ public class AppSchedulable extends Sche
@Override
public Resource getResourceUsage() {
- return this.app.getCurrentConsumption();
+ return app.getCurrentConsumption();
}
@@ -114,7 +114,7 @@ public class AppSchedulable extends Sche
* Get metrics reference from containing queue.
*/
public QueueMetrics getMetrics() {
- return this.queue.getQueueSchedulable().getMetrics();
+ return queue.getQueueSchedulable().getMetrics();
}
@Override
@@ -190,9 +190,9 @@ public class AppSchedulable extends Sche
RMContainer rmContainer = application.reserve(node, priority, null,
container);
node.reserveResource(application, priority, rmContainer);
- getMetrics().reserveResource(this.app.getUser(),
+ getMetrics().reserveResource(app.getUser(),
container.getResource());
- scheduler.getRootQueueMetrics().reserveResource(this.app.getUser(),
+ scheduler.getRootQueueMetrics().reserveResource(app.getUser(),
container.getResource());
}
@@ -257,13 +257,13 @@ public class AppSchedulable extends Sche
// TODO this should subtract resource just assigned
// TEMPROARY
getMetrics().setAvailableResourcesToQueue(
- this.scheduler.getClusterCapacity());
+ scheduler.getClusterCapacity());
}
// If we had previously made a reservation, delete it
if (reserved) {
- this.unreserve(application, priority, node);
+ unreserve(application, priority, node);
}
// Inform the node
@@ -290,7 +290,7 @@ public class AppSchedulable extends Sche
// Make sure the application still needs requests at this priority
if (app.getTotalRequiredResources(priority) == 0) {
- this.unreserve(app, priority, node);
+ unreserve(app, priority, node);
return Resources.none();
}
} else {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Sat Nov 10 00:49:15 2012
@@ -61,7 +61,7 @@ public class FSQueue {
queueSchedulable.addApp(appSchedulable);
}
- public void removeJob(FSSchedulerApp app) {
+ public void removeApp(FSSchedulerApp app) {
applications.remove(app);
queueSchedulable.removeApp(app);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueSchedulable.java Sat Nov 10 00:49:15 2012
@@ -80,7 +80,7 @@ public class FSQueueSchedulable extends
this.scheduler = scheduler;
this.queue = queue;
this.queueMgr = scheduler.getQueueManager();
- this.metrics = QueueMetrics.forQueue(this.getName(), null, true, scheduler.getConf());
+ this.metrics = QueueMetrics.forQueue(getName(), null, true, scheduler.getConf());
this.lastTimeAtMinShare = scheduler.getClock().getTime();
this.lastTimeAtHalfFairShare = scheduler.getClock().getTime();
}
@@ -113,7 +113,7 @@ public class FSQueueSchedulable extends
Resource toAdd = sched.getDemand();
if (LOG.isDebugEnabled()) {
LOG.debug("Counting resource from " + sched.getName() + " " + toAdd
- + "; Total resource consumption for " + this.getName() + " now "
+ + "; Total resource consumption for " + getName() + " now "
+ demand);
}
demand = Resources.add(demand, toAdd);
@@ -123,7 +123,7 @@ public class FSQueueSchedulable extends
}
}
if (LOG.isDebugEnabled()) {
- LOG.debug("The updated demand for " + this.getName() + " is " + demand
+ LOG.debug("The updated demand for " + getName() + " is " + demand
+ "; the max is " + maxRes);
}
}
@@ -164,9 +164,9 @@ public class FSQueueSchedulable extends
@Override
public Resource assignContainer(FSSchedulerNode node, boolean reserved) {
- LOG.debug("Node offered to queue: " + this.getName() + " reserved: " + reserved);
+ LOG.debug("Node offered to queue: " + getName() + " reserved: " + reserved);
// If this queue is over its limit, reject
- if (Resources.greaterThan(this.getResourceUsage(),
+ if (Resources.greaterThan(getResourceUsage(),
queueMgr.getMaxResources(queue.getName()))) {
return Resources.none();
}
@@ -258,7 +258,7 @@ public class FSQueueSchedulable extends
@Override
public Map<QueueACL, AccessControlList> getQueueAcls() {
- Map<QueueACL, AccessControlList> acls = this.queueMgr.getQueueAcls(this.getName());
+ Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
return new HashMap<QueueACL, AccessControlList>(acls);
}
@@ -284,7 +284,7 @@ public class FSQueueSchedulable extends
recordFactory.newRecordInstance(QueueUserACLInfo.class);
List<QueueACL> operations = new ArrayList<QueueACL>();
for (QueueACL operation : QueueACL.values()) {
- Map<QueueACL, AccessControlList> acls = this.queueMgr.getQueueAcls(this.getName());
+ Map<QueueACL, AccessControlList> acls = queueMgr.getQueueAcls(getName());
if (acls.get(operation).isUserAllowed(user)) {
operations.add(operation);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java Sat Nov 10 00:49:15 2012
@@ -112,12 +112,12 @@ public class FSSchedulerApp extends Sche
}
public ApplicationId getApplicationId() {
- return this.appSchedulingInfo.getApplicationId();
+ return appSchedulingInfo.getApplicationId();
}
@Override
public ApplicationAttemptId getApplicationAttemptId() {
- return this.appSchedulingInfo.getApplicationAttemptId();
+ return appSchedulingInfo.getApplicationAttemptId();
}
public void setAppSchedulable(AppSchedulable appSchedulable) {
@@ -129,7 +129,7 @@ public class FSSchedulerApp extends Sche
}
public String getUser() {
- return this.appSchedulingInfo.getUser();
+ return appSchedulingInfo.getUser();
}
public synchronized void updateResourceRequests(
@@ -138,19 +138,19 @@ public class FSSchedulerApp extends Sche
}
public Map<String, ResourceRequest> getResourceRequests(Priority priority) {
- return this.appSchedulingInfo.getResourceRequests(priority);
+ return appSchedulingInfo.getResourceRequests(priority);
}
public int getNewContainerId() {
- return this.appSchedulingInfo.getNewContainerId();
+ return appSchedulingInfo.getNewContainerId();
}
public Collection<Priority> getPriorities() {
- return this.appSchedulingInfo.getPriorities();
+ return appSchedulingInfo.getPriorities();
}
public ResourceRequest getResourceRequest(Priority priority, String nodeAddress) {
- return this.appSchedulingInfo.getResourceRequest(priority, nodeAddress);
+ return appSchedulingInfo.getResourceRequest(priority, nodeAddress);
}
public synchronized int getTotalRequiredResources(Priority priority) {
@@ -158,7 +158,7 @@ public class FSSchedulerApp extends Sche
}
public Resource getResource(Priority priority) {
- return this.appSchedulingInfo.getResource(priority);
+ return appSchedulingInfo.getResource(priority);
}
/**
@@ -167,11 +167,11 @@ public class FSSchedulerApp extends Sche
*/
@Override
public boolean isPending() {
- return this.appSchedulingInfo.isPending();
+ return appSchedulingInfo.isPending();
}
public String getQueueName() {
- return this.appSchedulingInfo.getQueueName();
+ return appSchedulingInfo.getQueueName();
}
/**
@@ -185,7 +185,7 @@ public class FSSchedulerApp extends Sche
public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) {
// Cleanup all scheduling information
- this.appSchedulingInfo.stop(rmAppAttemptFinalState);
+ appSchedulingInfo.stop(rmAppAttemptFinalState);
}
@SuppressWarnings("unchecked")
@@ -196,7 +196,7 @@ public class FSSchedulerApp extends Sche
getRMContainer(containerId);
if (rmContainer == null) {
// Some unknown container sneaked into the system. Kill it.
- this.rmContext.getDispatcher().getEventHandler()
+ rmContext.getDispatcher().getEventHandler()
.handle(new RMNodeCleanContainerEvent(nodeId, containerId));
return;
}
@@ -272,7 +272,7 @@ public class FSSchedulerApp extends Sche
}
synchronized public void addSchedulingOpportunity(Priority priority) {
- this.schedulingOpportunities.setCount(priority,
+ schedulingOpportunities.setCount(priority,
schedulingOpportunities.count(priority) + 1);
}
@@ -282,19 +282,19 @@ public class FSSchedulerApp extends Sche
* successfully did so.
*/
synchronized public int getSchedulingOpportunities(Priority priority) {
- return this.schedulingOpportunities.count(priority);
+ return schedulingOpportunities.count(priority);
}
synchronized void resetReReservations(Priority priority) {
- this.reReservations.setCount(priority, 0);
+ reReservations.setCount(priority, 0);
}
synchronized void addReReservation(Priority priority) {
- this.reReservations.add(priority);
+ reReservations.add(priority);
}
synchronized public int getReReservations(Priority priority) {
- return this.reReservations.count(priority);
+ return reReservations.count(priority);
}
public synchronized int getNumReservedContainers(Priority priority) {
@@ -458,8 +458,8 @@ public class FSSchedulerApp extends Sche
* @param priority The priority of the container scheduled.
*/
synchronized public void resetSchedulingOpportunities(Priority priority) {
- this.lastScheduledContainer.put(priority, System.currentTimeMillis());
- this.schedulingOpportunities.setCount(priority, 0);
+ lastScheduledContainer.put(priority, System.currentTimeMillis());
+ schedulingOpportunities.setCount(priority, 0);
}
/**
@@ -494,14 +494,14 @@ public class FSSchedulerApp extends Sche
rackLocalityThreshold;
// Relax locality constraints once we've surpassed threshold.
- if (this.getSchedulingOpportunities(priority) > (numNodes * threshold)) {
+ if (getSchedulingOpportunities(priority) > (numNodes * threshold)) {
if (allowed.equals(NodeType.NODE_LOCAL)) {
allowedLocalityLevel.put(priority, NodeType.RACK_LOCAL);
- this.resetSchedulingOpportunities(priority);
+ resetSchedulingOpportunities(priority);
}
else if (allowed.equals(NodeType.RACK_LOCAL)) {
allowedLocalityLevel.put(priority, NodeType.OFF_SWITCH);
- this.resetSchedulingOpportunities(priority);
+ resetSchedulingOpportunities(priority);
}
}
return allowedLocalityLevel.get(priority);
@@ -512,7 +512,7 @@ public class FSSchedulerApp extends Sche
Priority priority, ResourceRequest request,
Container container) {
// Update allowed locality level
- NodeType allowed = this.allowedLocalityLevel.get(priority);
+ NodeType allowed = allowedLocalityLevel.get(priority);
if (allowed != null) {
if (allowed.equals(NodeType.OFF_SWITCH) &&
(type.equals(NodeType.NODE_LOCAL) ||
@@ -532,9 +532,9 @@ public class FSSchedulerApp extends Sche
}
// Create RMContainer
- RMContainer rmContainer = new RMContainerImpl(container, this
- .getApplicationAttemptId(), node.getNodeID(), this.rmContext
- .getDispatcher().getEventHandler(), this.rmContext
+ RMContainer rmContainer = new RMContainerImpl(container,
+ getApplicationAttemptId(), node.getNodeID(), rmContext
+ .getDispatcher().getEventHandler(), rmContext
.getContainerAllocationExpirer());
// Add it to allContainers list.
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java Sat Nov 10 00:49:15 2012
@@ -67,25 +67,25 @@ public class FSSchedulerNode extends Sch
}
public RMNode getRMNode() {
- return this.rmNode;
+ return rmNode;
}
public NodeId getNodeID() {
- return this.rmNode.getNodeID();
+ return rmNode.getNodeID();
}
public String getHttpAddress() {
- return this.rmNode.getHttpAddress();
+ return rmNode.getHttpAddress();
}
@Override
public String getHostName() {
- return this.rmNode.getHostName();
+ return rmNode.getHostName();
}
@Override
public String getRackName() {
- return this.rmNode.getRackName();
+ return rmNode.getRackName();
}
/**
@@ -112,17 +112,18 @@ public class FSSchedulerNode extends Sch
@Override
public synchronized Resource getAvailableResource() {
- return this.availableResource;
+ return availableResource;
}
@Override
public synchronized Resource getUsedResource() {
- return this.usedResource;
+ return usedResource;
}
private synchronized boolean isValidContainer(Container c) {
- if (launchedContainers.containsKey(c.getId()))
+ if (launchedContainers.containsKey(c.getId())) {
return true;
+ }
return false;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Sat Nov 10 00:49:15 2012
@@ -139,11 +139,11 @@ public class FairScheduler implements Re
public FairSchedulerConfiguration getConf() {
- return this.conf;
+ return conf;
}
public QueueManager getQueueManager() {
- return this.queueMgr;
+ return queueMgr;
}
public List<FSQueueSchedulable> getQueueSchedulables() {
@@ -183,36 +183,34 @@ public class FairScheduler implements Re
* fair shares, deficits, minimum slot allocations, and amount of used and
* required resources per job.
*/
- protected void update() {
- synchronized (this) {
- queueMgr.reloadAllocsIfNecessary(); // Relaod alloc file
- updateRunnability(); // Set job runnability based on user/queue limits
- updatePreemptionVariables(); // Determine if any queues merit preemption
+ protected synchronized void update() {
+ queueMgr.reloadAllocsIfNecessary(); // Relaod alloc file
+ updateRunnability(); // Set job runnability based on user/queue limits
+ updatePreemptionVariables(); // Determine if any queues merit preemption
- // Update demands of apps and queues
- for (FSQueue queue: queueMgr.getQueues()) {
- queue.getQueueSchedulable().updateDemand();
- }
-
- // Compute fair shares based on updated demands
- List<FSQueueSchedulable> queueScheds = this.getQueueSchedulables();
- SchedulingAlgorithms.computeFairShares(
- queueScheds, clusterCapacity);
+ // Update demands of apps and queues
+ for (FSQueue queue: queueMgr.getQueues()) {
+ queue.getQueueSchedulable().updateDemand();
+ }
- // Update queue metrics for this queue
- for (FSQueueSchedulable sched : queueScheds) {
- sched.getMetrics().setAvailableResourcesToQueue(sched.getFairShare());
- }
+ // Compute fair shares based on updated demands
+ List<FSQueueSchedulable> queueScheds = getQueueSchedulables();
+ SchedulingAlgorithms.computeFairShares(
+ queueScheds, clusterCapacity);
- // Use the computed shares to assign shares within each queue
- for (FSQueue queue: queueMgr.getQueues()) {
- queue.getQueueSchedulable().redistributeShare();
- }
+ // Update queue metrics for this queue
+ for (FSQueueSchedulable sched : queueScheds) {
+ sched.getMetrics().setAvailableResourcesToQueue(sched.getFairShare());
+ }
- // Update recorded capacity of root queue (child queues are updated
- // when fair share is calculated).
- rootMetrics.setAvailableResourcesToQueue(clusterCapacity);
+ // Use the computed shares to assign shares within each queue
+ for (FSQueue queue: queueMgr.getQueues()) {
+ queue.getQueueSchedulable().redistributeShare();
}
+
+ // Update recorded capacity of root queue (child queues are updated
+ // when fair share is calculated).
+ rootMetrics.setAvailableResourcesToQueue(clusterCapacity);
}
/**
@@ -257,17 +255,16 @@ public class FairScheduler implements Re
* have been below half their fair share for the fairSharePreemptionTimeout.
* If such queues exist, compute how many tasks of each type need to be
* preempted and then select the right ones using preemptTasks.
- *
- * This method computes and logs the number of tasks we want to preempt even
- * if preemption is disabled, for debugging purposes.
*/
protected void preemptTasksIfNecessary() {
- if (!preemptionEnabled)
+ if (!preemptionEnabled) {
return;
+ }
long curTime = clock.getTime();
- if (curTime - lastPreemptCheckTime < preemptionInterval)
+ if (curTime - lastPreemptCheckTime < preemptionInterval) {
return;
+ }
lastPreemptCheckTime = curTime;
Resource resToPreempt = Resources.none();
@@ -288,8 +285,9 @@ public class FairScheduler implements Re
* lowest priority to preempt.
*/
protected void preemptResources(List<FSQueueSchedulable> scheds, Resource toPreempt) {
- if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none()))
+ if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none())) {
return;
+ }
Map<RMContainer, FSSchedulerApp> apps =
new HashMap<RMContainer, FSSchedulerApp>();
@@ -330,7 +328,7 @@ public class FairScheduler implements Re
// TODO: Not sure if this ever actually adds this to the list of cleanup
// containers on the RMNode (see SchedulerNode.releaseContainer()).
- this.completedContainer(container, status, RMContainerEventType.KILL);
+ completedContainer(container, status, RMContainerEventType.KILL);
toPreempt = Resources.subtract(toPreempt,
container.getContainer().getResource());
@@ -413,7 +411,7 @@ public class FairScheduler implements Re
}
public RMContainerTokenSecretManager getContainerTokenSecretManager() {
- return this.rmContext.getContainerTokenSecretManager();
+ return rmContext.getContainerTokenSecretManager();
}
public double getAppWeight(AppSchedulable app) {
@@ -437,28 +435,28 @@ public class FairScheduler implements Re
@Override
public Resource getMinimumResourceCapability() {
- return this.minimumAllocation;
+ return minimumAllocation;
}
@Override
public Resource getMaximumResourceCapability() {
- return this.maximumAllocation;
+ return maximumAllocation;
}
public double getNodeLocalityThreshold() {
- return this.nodeLocalityThreshold;
+ return nodeLocalityThreshold;
}
public double getRackLocalityThreshold() {
- return this.rackLocalityThreshold;
+ return rackLocalityThreshold;
}
public Resource getClusterCapacity() {
- return this.clusterCapacity;
+ return clusterCapacity;
}
public Clock getClock() {
- return this.clock;
+ return clock;
}
protected void setClock(Clock clock) {
@@ -478,11 +476,11 @@ public class FairScheduler implements Re
addApplication(ApplicationAttemptId applicationAttemptId,
String queueName, String user) {
- FSQueue queue = this.queueMgr.getQueue(queueName);
+ FSQueue queue = queueMgr.getQueue(queueName);
FSSchedulerApp schedulerApp =
new FSSchedulerApp(applicationAttemptId, user,
- queue.getQueueSchedulable(), new ActiveUsersManager(this.getRootQueueMetrics()),
+ queue.getQueueSchedulable(), new ActiveUsersManager(getRootQueueMetrics()),
rmContext, null);
// Inforce ACLs
@@ -553,8 +551,8 @@ public class FairScheduler implements Re
application.stop(rmAppAttemptFinalState);
// Inform the queue
- FSQueue queue = this.queueMgr.getQueue(application.getQueue().getQueueName());
- queue.removeJob(application);
+ FSQueue queue = queueMgr.getQueue(application.getQueue().getQueueName());
+ queue.removeApp(application);
// Remove from our data-structure
applications.remove(applicationAttemptId);
@@ -600,7 +598,7 @@ public class FairScheduler implements Re
}
private synchronized void addNode(RMNode node) {
- this.nodes.put(node.getNodeID(), new FSSchedulerNode(node));
+ nodes.put(node.getNodeID(), new FSSchedulerNode(node));
Resources.addTo(clusterCapacity, node.getTotalCapability());
LOG.info("Added node " + node.getNodeAddress() +
@@ -608,7 +606,7 @@ public class FairScheduler implements Re
}
private synchronized void removeNode(RMNode rmNode) {
- FSSchedulerNode node = this.nodes.get(rmNode.getNodeID());
+ FSSchedulerNode node = nodes.get(rmNode.getNodeID());
Resources.subtractFrom(clusterCapacity, rmNode.getTotalCapability());
// Remove running containers
@@ -631,7 +629,7 @@ public class FairScheduler implements Re
RMContainerEventType.KILL);
}
- this.nodes.remove(rmNode.getNodeID());
+ nodes.remove(rmNode.getNodeID());
LOG.info("Removed node " + rmNode.getNodeAddress() +
" cluster capacity: " + clusterCapacity);
}
@@ -669,10 +667,8 @@ public class FairScheduler implements Re
}
synchronized (application) {
-
if (!ask.isEmpty()) {
-
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("allocate: pre-update" +
" applicationAttemptId=" + appAttemptId +
" application=" + application.getApplicationId());
@@ -686,7 +682,7 @@ public class FairScheduler implements Re
application.showRequests();
}
- if(LOG.isDebugEnabled()) {
+ if (LOG.isDebugEnabled()) {
LOG.debug("allocate:" +
" applicationAttemptId=" + appAttemptId +
" #ask=" + ask.size());
@@ -764,7 +760,7 @@ public class FairScheduler implements Re
int assignedContainers = 0;
while (true) {
// At most one task is scheduled each iteration of this loop
- List<FSQueueSchedulable> scheds = this.getQueueSchedulables();
+ List<FSQueueSchedulable> scheds = getQueueSchedulables();
Collections.sort(scheds, new SchedulingAlgorithms.FairShareComparator());
boolean assignedContainer = false;
for (FSQueueSchedulable sched : scheds) {
@@ -796,11 +792,11 @@ public class FairScheduler implements Re
@Override
public SchedulerAppReport getSchedulerAppInfo(
ApplicationAttemptId appAttemptId) {
- if (!this.applications.containsKey(appAttemptId)) {
+ if (!applications.containsKey(appAttemptId)) {
LOG.error("Request for appInfo of unknown attempt" + appAttemptId);
return null;
}
- return new SchedulerAppReport(this.applications.get(appAttemptId));
+ return new SchedulerAppReport(applications.get(appAttemptId));
}
@Override
@@ -812,37 +808,30 @@ public class FairScheduler implements Re
public void handle(SchedulerEvent event) {
switch(event.getType()) {
case NODE_ADDED:
- {
if (!(event instanceof NodeAddedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)event;
addNode(nodeAddedEvent.getAddedRMNode());
- }
- break;
+ break;
case NODE_REMOVED:
- {
if (!(event instanceof NodeRemovedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)event;
removeNode(nodeRemovedEvent.getRemovedRMNode());
- }
- break;
+ break;
case NODE_UPDATE:
- {
if (!(event instanceof NodeUpdateSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
NodeUpdateSchedulerEvent nodeUpdatedEvent =
(NodeUpdateSchedulerEvent)event;
- this.nodeUpdate(nodeUpdatedEvent.getRMNode(),
+ nodeUpdate(nodeUpdatedEvent.getRMNode(),
nodeUpdatedEvent.getNewlyLaunchedContainers(),
nodeUpdatedEvent.getCompletedContainers());
- }
- break;
+ break;
case APP_ADDED:
- {
if (!(event instanceof AppAddedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
@@ -857,20 +846,16 @@ public class FairScheduler implements Re
addApplication(appAddedEvent.getApplicationAttemptId(), queue,
appAddedEvent.getUser());
- }
- break;
+ break;
case APP_REMOVED:
- {
if (!(event instanceof AppRemovedSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)event;
- this.removeApplication(appRemovedEvent.getApplicationAttemptID(),
+ removeApplication(appRemovedEvent.getApplicationAttemptID(),
appRemovedEvent.getFinalAttemptState());
- }
- break;
+ break;
case CONTAINER_EXPIRED:
- {
if (!(event instanceof ContainerExpiredSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
}
@@ -882,8 +867,7 @@ public class FairScheduler implements Re
containerId,
SchedulerUtils.EXPIRED_CONTAINER),
RMContainerEventType.EXPIRE);
- }
- break;
+ break;
default:
LOG.error("Unknown event arrived at FairScheduler: " + event.toString());
}
@@ -897,9 +881,9 @@ public class FairScheduler implements Re
@Override
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
- if (!this.initialized) {
+ if (!initialized) {
this.conf = new FairSchedulerConfiguration(conf);
- this.rootMetrics = QueueMetrics.forQueue("root", null, true, conf);
+ rootMetrics = QueueMetrics.forQueue("root", null, true, conf);
this.rmContext = rmContext;
this.clock = new SystemClock();
this.eventLog = new FairSchedulerEventLog();
@@ -973,7 +957,7 @@ public class FairScheduler implements Re
@Override
public int getNumClusterNodes() {
- return this.nodes.size();
+ return nodes.size();
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Sat Nov 10 00:49:15 2012
@@ -202,7 +202,7 @@ public class QueueManager {
* Get the queue for a given AppSchedulable.
*/
public FSQueue getQueueForApp(AppSchedulable app) {
- return this.getQueue(app.getApp().getQueueName());
+ return getQueue(app.getApp().getQueueName());
}
/**
@@ -388,7 +388,7 @@ public class QueueManager {
// Commit the reload; also create any queue defined in the alloc file
// if it does not already exist, so it can be displayed on the web UI.
- synchronized(this) {
+ synchronized (this) {
setMinResources(minQueueResources);
setMaxResources(maxQueueResources);
setQueueMaxApps(queueMaxApps);
@@ -431,14 +431,14 @@ public class QueueManager {
synchronized(minQueueResourcesMO) {
if (minQueueResources.containsKey(queue)) {
return minQueueResources.get(queue);
- } else{
+ } else {
return Resources.createResource(0);
}
}
}
private void setMinResources(Map<String, Resource> resources) {
- synchronized(minQueueResourcesMO) {
+ synchronized (minQueueResourcesMO) {
minQueueResources = resources;
}
}
@@ -457,7 +457,7 @@ public class QueueManager {
}
private void setMaxResources(Map<String, Resource> resources) {
- synchronized(maxQueueResourcesMO) {
+ synchronized (maxQueueResourcesMO) {
maxQueueResources = resources;
}
}
@@ -472,8 +472,8 @@ public class QueueManager {
/**
* Remove an app
*/
- public synchronized void removeJob(FSSchedulerApp app) {
- getQueue(app.getQueueName()).removeJob(app);
+ public synchronized void removeApp(FSSchedulerApp app) {
+ getQueue(app.getQueueName()).removeApp(app);
}
/**
@@ -543,7 +543,7 @@ public class QueueManager {
}
private int getQueueMaxAppsDefault(){
- synchronized(queueMaxAppsDefaultMO) {
+ synchronized (queueMaxAppsDefaultMO) {
return queueMaxAppsDefault;
}
}
@@ -575,11 +575,12 @@ public class QueueManager {
queueWeights = weights;
}
}
+
/**
- * Get a queue's min share preemption timeout, in milliseconds. This is the
- * time after which jobs in the queue may kill other queues' tasks if they
- * are below their min share.
- */
+ * Get a queue's min share preemption timeout, in milliseconds. This is the
+ * time after which jobs in the queue may kill other queues' tasks if they
+ * are below their min share.
+ */
public long getMinSharePreemptionTimeout(String queueName) {
synchronized (minSharePreemptionTimeoutsMO) {
if (minSharePreemptionTimeouts.containsKey(queueName)) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java Sat Nov 10 00:49:15 2012
@@ -24,7 +24,7 @@ import org.apache.hadoop.conf.Configurab
/**
* A pluggable object for altering the weights of apps in the fair scheduler,
- * which is used for example by {@link NewJobWeightBooster} to give higher
+ * which is used for example by {@link NewAppWeightBooster} to give higher
* weight to new jobs so that short jobs finish faster.
*
* May implement {@link Configurable} to access configuration parameters.
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java Sat Nov 10 00:49:15 2012
@@ -81,38 +81,38 @@ public class TestApplicationCleanup {
new ArrayList<ContainerId>()).getAllocatedContainers();
int contReceived = conts.size();
int waitCount = 0;
- while (contReceived < request && waitCount++ < 20) {
+ while (contReceived < request && waitCount++ < 200) {
+ LOG.info("Got " + contReceived + " containers. Waiting to get "
+ + request);
+ Thread.sleep(100);
conts = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
contReceived += conts.size();
- LOG.info("Got " + contReceived + " containers. Waiting to get "
- + request);
- Thread.sleep(2000);
}
- Assert.assertEquals(request, conts.size());
+ Assert.assertEquals(request, contReceived);
am.unregisterAppAttempt();
HeartbeatResponse resp = nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1,
ContainerState.COMPLETE);
am.waitForState(RMAppAttemptState.FINISHED);
- int cleanedConts = 0;
- int cleanedApps = 0;
- List<ContainerId> contsToClean = null;
- List<ApplicationId> apps = null;
-
//currently only containers are cleaned via this
//AM container is cleaned via container launcher
+ resp = nm1.nodeHeartbeat(true);
+ List<ContainerId> contsToClean = resp.getContainersToCleanupList();
+ List<ApplicationId> apps = resp.getApplicationsToCleanupList();
+ int cleanedConts = contsToClean.size();
+ int cleanedApps = apps.size();
waitCount = 0;
- while ((cleanedConts < 2 || cleanedApps < 1) && waitCount++ < 20) {
- contsToClean = resp.getContainersToCleanupList();
- apps = resp.getApplicationsToCleanupList();
+ while ((cleanedConts < 2 || cleanedApps < 1) && waitCount++ < 200) {
LOG.info("Waiting to get cleanup events.. cleanedConts: "
+ cleanedConts + " cleanedApps: " + cleanedApps);
+ Thread.sleep(100);
+ resp = nm1.nodeHeartbeat(true);
+ contsToClean = resp.getContainersToCleanupList();
+ apps = resp.getApplicationsToCleanupList();
cleanedConts += contsToClean.size();
cleanedApps += apps.size();
- Thread.sleep(1000);
- resp = nm1.nodeHeartbeat(true);
}
Assert.assertEquals(1, apps.size());
@@ -170,20 +170,20 @@ public class TestApplicationCleanup {
new ArrayList<ContainerId>()).getAllocatedContainers();
int contReceived = conts.size();
int waitCount = 0;
- while (contReceived < request && waitCount++ < 20) {
+ while (contReceived < request && waitCount++ < 200) {
+ LOG.info("Got " + contReceived + " containers. Waiting to get "
+ + request);
+ Thread.sleep(100);
conts = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
dispatcher.await();
contReceived += conts.size();
- LOG.info("Got " + contReceived + " containers. Waiting to get "
- + request);
- Thread.sleep(2000);
}
- Assert.assertEquals(request, conts.size());
+ Assert.assertEquals(request, contReceived);
// Release a container.
ArrayList<ContainerId> release = new ArrayList<ContainerId>();
- release.add(conts.get(1).getId());
+ release.add(conts.get(0).getId());
am.allocate(new ArrayList<ResourceRequest>(), release);
dispatcher.await();
@@ -194,7 +194,7 @@ public class TestApplicationCleanup {
new HashMap<ApplicationId, List<ContainerStatus>>();
ArrayList<ContainerStatus> containerStatusList =
new ArrayList<ContainerStatus>();
- containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(1)
+ containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
.getId(), ContainerState.RUNNING, "nothing", 0));
containerStatuses.put(app.getApplicationId(), containerStatusList);
@@ -203,13 +203,13 @@ public class TestApplicationCleanup {
List<ContainerId> contsToClean = resp.getContainersToCleanupList();
int cleanedConts = contsToClean.size();
waitCount = 0;
- while (cleanedConts < 1 && waitCount++ < 20) {
+ while (cleanedConts < 1 && waitCount++ < 200) {
+ LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
+ Thread.sleep(100);
resp = nm1.nodeHeartbeat(true);
dispatcher.await();
contsToClean = resp.getContainersToCleanupList();
- LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
cleanedConts += contsToClean.size();
- Thread.sleep(1000);
}
LOG.info("Got cleanup for " + contsToClean.get(0));
Assert.assertEquals(1, cleanedConts);
@@ -220,7 +220,7 @@ public class TestApplicationCleanup {
+ "NM getting cleanup");
containerStatuses.clear();
containerStatusList.clear();
- containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(1)
+ containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(0)
.getId(), ContainerState.RUNNING, "nothing", 0));
containerStatuses.put(app.getApplicationId(), containerStatusList);
@@ -231,13 +231,13 @@ public class TestApplicationCleanup {
// The cleanup list won't be instantaneous as it is given out by scheduler
// and not RMNodeImpl.
waitCount = 0;
- while (cleanedConts < 1 && waitCount++ < 20) {
+ while (cleanedConts < 1 && waitCount++ < 200) {
+ LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
+ Thread.sleep(100);
resp = nm1.nodeHeartbeat(true);
dispatcher.await();
contsToClean = resp.getContainersToCleanupList();
- LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
cleanedConts += contsToClean.size();
- Thread.sleep(1000);
}
LOG.info("Got cleanup for " + contsToClean.get(0));
Assert.assertEquals(1, cleanedConts);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Sat Nov 10 00:49:15 2012
@@ -1259,7 +1259,7 @@ public class TestLeafQueue {
assignment = a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
- assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
+ assertEquals(4, app_0.getSchedulingOpportunities(priority)); // should NOT reset
assertEquals(2, app_0.getTotalRequiredResources(priority));
assertEquals(NodeType.OFF_SWITCH, assignment.getType());
@@ -1408,11 +1408,11 @@ public class TestLeafQueue {
assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
assertEquals(1, app_0.getTotalRequiredResources(priority_2));
- // Another off-switch, shouldn allocate OFF_SWITCH P1
+ // Another off-switch, shouldn't allocate OFF_SWITCH P1
a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
eq(priority_1), any(ResourceRequest.class), any(Container.class));
- assertEquals(0, app_0.getSchedulingOpportunities(priority_1));
+ assertEquals(3, app_0.getSchedulingOpportunities(priority_1));
assertEquals(1, app_0.getTotalRequiredResources(priority_1));
verify(app_0, never()).allocate(any(NodeType.class), eq(node_2),
eq(priority_2), any(ResourceRequest.class), any(Container.class));
@@ -1438,7 +1438,7 @@ public class TestLeafQueue {
assertEquals(0, app_0.getTotalRequiredResources(priority_1));
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_1),
eq(priority_2), any(ResourceRequest.class), any(Container.class));
- assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
+ assertEquals(1, app_0.getSchedulingOpportunities(priority_2));
assertEquals(0, app_0.getTotalRequiredResources(priority_2));
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm?rev=1407706&r1=1407705&r2=1407706&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/index.apt.vm Sat Nov 10 00:49:15 2012
@@ -49,7 +49,7 @@ MapReduce NextGen aka YARN aka MRv2
* {{{./WebApplicationProxy.html}Web Application Proxy}}
- * {{{./CLIMiniCluster.html}CLI MiniCluster}}
+ * {{{../../hadoop-project-dist/hadoop-common/CLIMiniCluster.html}CLI MiniCluster}}
- * {{{./EncryptedShuffle.html}Encrypted Shuffle}}
+ * {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/EncryptedShuffle.html}Encrypted Shuffle}}