You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cl...@apache.org on 2014/05/30 00:27:33 UTC
svn commit: r1598435 [2/2] - in
/hadoop/common/branches/fs-encryption/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ ...
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java Thu May 29 22:27:25 2014
@@ -33,10 +33,10 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
@Private
@Unstable
@@ -209,6 +209,36 @@ public class FSLeafQueue extends FSQueue
}
@Override
+ public RMContainer preemptContainer() {
+ RMContainer toBePreempted = null;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Queue " + getName() + " is going to preempt a container " +
+ "from its applications.");
+ }
+
+ // If this queue is not over its fair share, reject
+ if (!preemptContainerPreCheck()) {
+ return toBePreempted;
+ }
+
+ // Choose the app that is most over fair share
+ Comparator<Schedulable> comparator = policy.getComparator();
+ AppSchedulable candidateSched = null;
+ for (AppSchedulable sched : runnableAppScheds) {
+ if (candidateSched == null ||
+ comparator.compare(sched, candidateSched) > 0) {
+ candidateSched = sched;
+ }
+ }
+
+ // Preempt from the selected app
+ if (candidateSched != null) {
+ toBePreempted = candidateSched.preemptContainer();
+ }
+ return toBePreempted;
+ }
+
+ @Override
public List<FSQueue> getChildQueues() {
return new ArrayList<FSQueue>(1);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Thu May 29 22:27:25 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
@@ -157,6 +159,32 @@ public class FSParentQueue extends FSQue
}
@Override
+ public RMContainer preemptContainer() {
+ RMContainer toBePreempted = null;
+
+ // If this queue is not over its fair share, reject
+ if (!preemptContainerPreCheck()) {
+ return toBePreempted;
+ }
+
+ // Find the childQueue which is most over fair share
+ FSQueue candidateQueue = null;
+ Comparator<Schedulable> comparator = policy.getComparator();
+ for (FSQueue queue : childQueues) {
+ if (candidateQueue == null ||
+ comparator.compare(queue, candidateQueue) > 0) {
+ candidateQueue = queue;
+ }
+ }
+
+ // Let the selected queue choose which of its container to preempt
+ if (candidateQueue != null) {
+ toBePreempted = candidateQueue.preemptContainer();
+ }
+ return toBePreempted;
+ }
+
+ @Override
public List<FSQueue> getChildQueues() {
return childQueues;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Thu May 29 22:27:25 2014
@@ -187,4 +187,17 @@ public abstract class FSQueue extends Sc
}
return true;
}
+
+ /**
+ * Helper method to check if the queue should preempt containers
+ *
+ * @return true if check passes (can preempt) or false otherwise
+ */
+ protected boolean preemptContainerPreCheck() {
+ if (this == scheduler.getQueueManager().getRootQueue()) {
+ return true;
+ }
+ return parent.getPolicy()
+ .checkIfUsageOverFairShare(getResourceUsage(), getFairShare());
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java Thu May 29 22:27:25 2014
@@ -59,6 +59,8 @@ public class FSSchedulerApp extends Sche
private AppSchedulable appSchedulable;
final Map<RMContainer, Long> preemptionMap = new HashMap<RMContainer, Long>();
+
+ private Resource preemptedResources = Resources.createResource(0);
public FSSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, FSLeafQueue queue, ActiveUsersManager activeUsersManager,
@@ -316,6 +318,7 @@ public class FSSchedulerApp extends Sche
public void addPreemption(RMContainer container, long time) {
assert preemptionMap.get(container) == null;
preemptionMap.put(container, time);
+ Resources.addTo(preemptedResources, container.getAllocatedResource());
}
public Long getContainerPreemptionTime(RMContainer container) {
@@ -330,4 +333,20 @@ public class FSSchedulerApp extends Sche
public FSLeafQueue getQueue() {
return (FSLeafQueue)super.getQueue();
}
+
+ public Resource getPreemptedResources() {
+ return preemptedResources;
+ }
+
+ public void resetPreemptedResources() {
+ preemptedResources = Resources.createResource(0);
+ for (RMContainer container : getPreemptionContainers()) {
+ Resources.addTo(preemptedResources, container.getAllocatedResource());
+ }
+ }
+
+ public void clearPreemptedResources() {
+ preemptedResources.setMemory(0);
+ preemptedResources.setVirtualCores(0);
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Thu May 29 22:27:25 2014
@@ -20,14 +20,11 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -148,7 +145,11 @@ public class FairScheduler extends
// Time we last ran preemptTasksIfNecessary
private long lastPreemptCheckTime;
- // How often tasks are preempted
+ // Preemption related variables
+ protected boolean preemptionEnabled;
+ protected float preemptionUtilizationThreshold;
+
+ // How often tasks are preempted
protected long preemptionInterval;
// ms to wait before force killing stuff (must be longer than a couple
@@ -158,7 +159,6 @@ public class FairScheduler extends
// Containers whose AMs have been warned that they will be preempted soon.
private List<RMContainer> warnedContainers = new ArrayList<RMContainer>();
- protected boolean preemptionEnabled;
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster
protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not
@@ -318,7 +318,7 @@ public class FairScheduler extends
* and then select the right ones using preemptTasks.
*/
protected synchronized void preemptTasksIfNecessary() {
- if (!preemptionEnabled) {
+ if (!shouldAttemptPreemption()) {
return;
}
@@ -328,101 +328,84 @@ public class FairScheduler extends
}
lastPreemptCheckTime = curTime;
- Resource resToPreempt = Resources.none();
-
+ Resource resToPreempt = Resources.clone(Resources.none());
for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
- resToPreempt = Resources.add(resToPreempt, resToPreempt(sched, curTime));
+ Resources.addTo(resToPreempt, resToPreempt(sched, curTime));
}
if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt,
Resources.none())) {
- preemptResources(queueMgr.getLeafQueues(), resToPreempt);
+ preemptResources(resToPreempt);
}
}
/**
- * Preempt a quantity of resources from a list of QueueSchedulables. The
- * policy for this is to pick apps from queues that are over their fair share,
- * but make sure that no queue is placed below its fair share in the process.
- * We further prioritize preemption by choosing containers with lowest
- * priority to preempt.
+ * Preempt a quantity of resources. Each round, we start from the root queue,
+ * level-by-level, until choosing a candidate application.
+ * The policy for prioritizing preemption for each queue depends on its
+ * SchedulingPolicy: (1) fairshare/DRF, choose the ChildSchedulable that is
+ * most over its fair share; (2) FIFO, choose the childSchedulable that is
+ * latest launched.
+ * Inside each application, we further prioritize preemption by choosing
+ * containers with lowest priority to preempt.
+ * We make sure that no queue is placed below its fair share in the process.
*/
- protected void preemptResources(Collection<FSLeafQueue> scheds,
- Resource toPreempt) {
- if (scheds.isEmpty() || Resources.equals(toPreempt, Resources.none())) {
+ protected void preemptResources(Resource toPreempt) {
+ if (Resources.equals(toPreempt, Resources.none())) {
return;
}
- Map<RMContainer, FSSchedulerApp> apps =
- new HashMap<RMContainer, FSSchedulerApp>();
- Map<RMContainer, FSLeafQueue> queues =
- new HashMap<RMContainer, FSLeafQueue>();
-
- // Collect running containers from over-scheduled queues
- List<RMContainer> runningContainers = new ArrayList<RMContainer>();
- for (FSLeafQueue sched : scheds) {
- if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- sched.getResourceUsage(), sched.getFairShare())) {
- for (AppSchedulable as : sched.getRunnableAppSchedulables()) {
- for (RMContainer c : as.getApp().getLiveContainers()) {
- runningContainers.add(c);
- apps.put(c, as.getApp());
- queues.put(c, sched);
- }
- }
- }
- }
-
- // Sort containers into reverse order of priority
- Collections.sort(runningContainers, new Comparator<RMContainer>() {
- public int compare(RMContainer c1, RMContainer c2) {
- int ret = c1.getContainer().getPriority().compareTo(
- c2.getContainer().getPriority());
- if (ret == 0) {
- return c2.getContainerId().compareTo(c1.getContainerId());
- }
- return ret;
- }
- });
-
// Scan down the list of containers we've already warned and kill them
// if we need to. Remove any containers from the list that we don't need
// or that are no longer running.
Iterator<RMContainer> warnedIter = warnedContainers.iterator();
- Set<RMContainer> preemptedThisRound = new HashSet<RMContainer>();
while (warnedIter.hasNext()) {
RMContainer container = warnedIter.next();
- if (container.getState() == RMContainerState.RUNNING &&
+ if ((container.getState() == RMContainerState.RUNNING ||
+ container.getState() == RMContainerState.ALLOCATED) &&
Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
toPreempt, Resources.none())) {
- warnOrKillContainer(container, apps.get(container), queues.get(container));
- preemptedThisRound.add(container);
+ warnOrKillContainer(container);
Resources.subtractFrom(toPreempt, container.getContainer().getResource());
} else {
warnedIter.remove();
}
}
- // Scan down the rest of the containers until we've preempted enough, making
- // sure we don't preempt too many from any queue
- Iterator<RMContainer> runningIter = runningContainers.iterator();
- while (runningIter.hasNext() &&
- Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- toPreempt, Resources.none())) {
- RMContainer container = runningIter.next();
- FSLeafQueue sched = queues.get(container);
- if (!preemptedThisRound.contains(container) &&
- Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
- sched.getResourceUsage(), sched.getFairShare())) {
- warnOrKillContainer(container, apps.get(container), sched);
-
- warnedContainers.add(container);
- Resources.subtractFrom(toPreempt, container.getContainer().getResource());
+ try {
+ // Reset preemptedResource for each app
+ for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
+ for (AppSchedulable app : queue.getRunnableAppSchedulables()) {
+ app.getApp().resetPreemptedResources();
+ }
+ }
+
+ while (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource,
+ toPreempt, Resources.none())) {
+ RMContainer container =
+ getQueueManager().getRootQueue().preemptContainer();
+ if (container == null) {
+ break;
+ } else {
+ warnOrKillContainer(container);
+ warnedContainers.add(container);
+ Resources.subtractFrom(
+ toPreempt, container.getContainer().getResource());
+ }
+ }
+ } finally {
+ // Clear preemptedResources for each app
+ for (FSLeafQueue queue : getQueueManager().getLeafQueues()) {
+ for (AppSchedulable app : queue.getRunnableAppSchedulables()) {
+ app.getApp().clearPreemptedResources();
+ }
}
}
}
- private void warnOrKillContainer(RMContainer container, FSSchedulerApp app,
- FSLeafQueue queue) {
+ private void warnOrKillContainer(RMContainer container) {
+ ApplicationAttemptId appAttemptId = container.getApplicationAttemptId();
+ FSSchedulerApp app = getSchedulerApp(appAttemptId);
+ FSLeafQueue queue = app.getQueue();
LOG.info("Preempting container (prio=" + container.getContainer().getPriority() +
"res=" + container.getContainer().getResource() +
") from queue " + queue.getName());
@@ -1067,6 +1050,22 @@ public class FairScheduler extends
clusterResource, rootMetrics.getAllocatedResources()));
}
+ /**
+ * Check if preemption is enabled and the utilization threshold for
+ * preemption is met.
+ *
+ * @return true if preemption should be attempted, false otherwise.
+ */
+ private boolean shouldAttemptPreemption() {
+ if (preemptionEnabled) {
+ return (preemptionUtilizationThreshold < Math.max(
+ (float) rootMetrics.getAvailableMB() / clusterResource.getMemory(),
+ (float) rootMetrics.getAvailableVirtualCores() /
+ clusterResource.getVirtualCores()));
+ }
+ return false;
+ }
+
@Override
public QueueMetrics getRootQueueMetrics() {
return rootMetrics;
@@ -1172,6 +1171,8 @@ public class FairScheduler extends
nodeLocalityDelayMs = this.conf.getLocalityDelayNodeMs();
rackLocalityDelayMs = this.conf.getLocalityDelayRackMs();
preemptionEnabled = this.conf.getPreemptionEnabled();
+ preemptionUtilizationThreshold =
+ this.conf.getPreemptionUtilizationThreshold();
assignMultiple = this.conf.getAssignMultiple();
maxAssign = this.conf.getMaxAssign();
sizeBasedWeight = this.conf.getSizeBasedWeight();
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java Thu May 29 22:27:25 2014
@@ -101,6 +101,10 @@ public class FairSchedulerConfiguration
/** Whether preemption is enabled. */
protected static final String PREEMPTION = CONF_PREFIX + "preemption";
protected static final boolean DEFAULT_PREEMPTION = false;
+
+ protected static final String PREEMPTION_THRESHOLD =
+ CONF_PREFIX + "preemption.cluster-utilization-threshold";
+ protected static final float DEFAULT_PREEMPTION_THRESHOLD = 0.8f;
protected static final String PREEMPTION_INTERVAL = CONF_PREFIX + "preemptionInterval";
protected static final int DEFAULT_PREEMPTION_INTERVAL = 5000;
@@ -185,6 +189,10 @@ public class FairSchedulerConfiguration
return getBoolean(PREEMPTION, DEFAULT_PREEMPTION);
}
+ public float getPreemptionUtilizationThreshold() {
+ return getFloat(PREEMPTION_THRESHOLD, DEFAULT_PREEMPTION_THRESHOLD);
+ }
+
public boolean getAssignMultiple() {
return getBoolean(ASSIGN_MULTIPLE, DEFAULT_ASSIGN_MULTIPLE);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueuePlacementRule.java Thu May 29 22:27:25 2014
@@ -276,21 +276,48 @@ public abstract class QueuePlacementRule
}
/**
- * Places all apps in the default queue
+ * Places apps in the specified default queue. If no default queue is
+ * specified the app is placed in root.default queue.
*/
public static class Default extends QueuePlacementRule {
+ @VisibleForTesting
+ String defaultQueueName;
+
+ @Override
+ public QueuePlacementRule initialize(boolean create,
+ Map<String, String> args) {
+ if (defaultQueueName == null) {
+ defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+ }
+ return super.initialize(create, args);
+ }
+
+ @Override
+ public void initializeFromXml(Element el)
+ throws AllocationConfigurationException {
+ defaultQueueName = el.getAttribute("queue");
+ if (defaultQueueName != null && !defaultQueueName.isEmpty()) {
+ if (!defaultQueueName.startsWith("root.")) {
+ defaultQueueName = "root." + defaultQueueName;
+ }
+ } else {
+ defaultQueueName = "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+ }
+ super.initializeFromXml(el);
+ }
+
@Override
protected String getQueueForApp(String requestedQueue, String user,
Groups groups, Map<FSQueueType, Set<String>> configuredQueues) {
- return "root." + YarnConfiguration.DEFAULT_QUEUE_NAME;
+ return defaultQueueName;
}
-
+
@Override
public boolean isTerminal() {
return true;
}
}
-
+
/**
* Rejects all apps
*/
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java Thu May 29 22:27:25 2014
@@ -23,6 +23,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -100,6 +101,11 @@ public abstract class Schedulable {
*/
public abstract Resource assignContainer(FSSchedulerNode node);
+ /**
+ * Preempt a container from this Schedulable if possible.
+ */
+ public abstract RMContainer preemptContainer();
+
/** Assign a fair share to this Schedulable. */
public void setFairShare(Resource fairShare) {
this.fairShare = fairShare;
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java Thu May 29 22:27:25 2014
@@ -139,4 +139,14 @@ public abstract class SchedulingPolicy {
*/
public abstract void computeShares(
Collection<? extends Schedulable> schedulables, Resource totalResources);
+
+ /**
+ * Check if the resource usage is over the fair share under this policy
+ *
+ * @param usage {@link Resource} the resource usage
+ * @param fairShare {@link Resource} the fair share
+ * @return true if check passes (is over) or false otherwise
+ */
+ public abstract boolean checkIfUsageOverFairShare(
+ Resource usage, Resource fairShare);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java Thu May 29 22:27:25 2014
@@ -70,6 +70,11 @@ public class DominantResourceFairnessPol
}
@Override
+ public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
+ return !Resources.fitsIn(usage, fairShare);
+ }
+
+ @Override
public void initialize(Resource clusterCapacity) {
comparator.setClusterCapacity(clusterCapacity);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java Thu May 29 22:27:25 2014
@@ -120,6 +120,11 @@ public class FairSharePolicy extends Sch
}
@Override
+ public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
+ return Resources.greaterThan(RESOURCE_CALCULATOR, null, usage, fairShare);
+ }
+
+ @Override
public byte getApplicableDepth() {
return SchedulingPolicy.DEPTH_ANY;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java Thu May 29 22:27:25 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.re
import java.io.Serializable;
import java.util.Collection;
import java.util.Comparator;
-import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -89,6 +88,13 @@ public class FifoPolicy extends Scheduli
}
@Override
+ public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
+ throw new UnsupportedOperationException(
+ "FifoPolicy doesn't support checkIfUsageOverFairshare operation, " +
+ "as FifoPolicy only works for FSLeafQueue.");
+ }
+
+ @Override
public byte getApplicableDepth() {
return SchedulingPolicy.DEPTH_LEAF;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java Thu May 29 22:27:25 2014
@@ -348,14 +348,14 @@ public class TestRMAdminService {
rm.adminService.refreshSuperUserGroupsConfiguration(
RefreshSuperUserGroupsConfigurationRequest.newInstance());
- Assert.assertTrue(ProxyUsers.getProxyGroups()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
.get("hadoop.proxyuser.test.groups").size() == 1);
- Assert.assertTrue(ProxyUsers.getProxyGroups()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
.get("hadoop.proxyuser.test.groups").contains("test_groups"));
- Assert.assertTrue(ProxyUsers.getProxyHosts()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").size() == 1);
- Assert.assertTrue(ProxyUsers.getProxyHosts()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
}
@@ -708,14 +708,14 @@ public class TestRMAdminService {
aclsString);
// verify ProxyUsers and ProxyHosts
- Assert.assertTrue(ProxyUsers.getProxyGroups()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
.get("hadoop.proxyuser.test.groups").size() == 1);
- Assert.assertTrue(ProxyUsers.getProxyGroups()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyGroups()
.get("hadoop.proxyuser.test.groups").contains("test_groups"));
- Assert.assertTrue(ProxyUsers.getProxyHosts()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").size() == 1);
- Assert.assertTrue(ProxyUsers.getProxyHosts()
+ Assert.assertTrue(ProxyUsers.getDefaultImpersonationProvider().getProxyHosts()
.get("hadoop.proxyuser.test.hosts").contains("test_hosts"));
// verify UserToGroupsMappings
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Thu May 29 22:27:25 2014
@@ -1638,10 +1638,16 @@ public class TestRMRestart {
// create app that gets launched and does allocate before RM restart
RMApp app1 = rm1.submitApp(200);
- assertQueueMetrics(qm1, 1, 1, 0, 0);
- nm1.nodeHeartbeat(true);
+ // Need to wait first for AppAttempt to be started (RMAppState.ACCEPTED)
+ // and then for it to reach RMAppAttemptState.SCHEDULED
+ // inorder to ensure appsPending metric is incremented
+ rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
+ rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
+ assertQueueMetrics(qm1, 1, 1, 0, 0);
+
+ nm1.nodeHeartbeat(true);
rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
MockAM am1 = rm1.sendAMLaunched(attempt1.getAppAttemptId());
am1.registerAppAttempt();
@@ -1660,11 +1666,12 @@ public class TestRMRestart {
// PHASE 2: create new RM and start from old state
// create new RM to represent restart and recover state
MockRM rm2 = new MockRM(conf, memStore);
- rm2.start();
- nm1.setResourceTrackerService(rm2.getResourceTrackerService());
QueueMetrics qm2 = rm2.getResourceScheduler().getRootQueueMetrics();
resetQueueMetrics(qm2);
assertQueueMetrics(qm2, 0, 0, 0, 0);
+
+ rm2.start();
+ nm1.setResourceTrackerService(rm2.getResourceTrackerService());
// recover app
RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());
am1.setAMRMProtocol(rm2.getApplicationMasterService());
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java Thu May 29 22:27:25 2014
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -84,6 +85,11 @@ public class FakeSchedulable extends Sch
}
@Override
+ public RMContainer preemptContainer() {
+ return null;
+ }
+
+ @Override
public Resource getDemand() {
return null;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu May 29 22:27:25 2014
@@ -55,7 +55,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
@@ -63,8 +62,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -91,10 +88,10 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.Default;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
-import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Assert;
@@ -105,46 +102,14 @@ import org.xml.sax.SAXException;
import com.google.common.collect.Sets;
@SuppressWarnings("unchecked")
-public class TestFairScheduler {
+public class TestFairScheduler extends FairSchedulerTestBase {
+ private final static String ALLOC_FILE =
+ new File(TEST_DIR, "test-queues").getAbsolutePath();
- static class MockClock implements Clock {
- private long time = 0;
- @Override
- public long getTime() {
- return time;
- }
-
- public void tick(int seconds) {
- time = time + seconds * 1000;
- }
-
- }
-
- final static String TEST_DIR = new File(System.getProperty("test.build.data",
- "/tmp")).getAbsolutePath();
-
- final static String ALLOC_FILE = new File(TEST_DIR,
- "test-queues").getAbsolutePath();
-
- private FairScheduler scheduler;
- private ResourceManager resourceManager;
- private Configuration conf;
- private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
- private int APP_ID = 1; // Incrementing counter for schedling apps
- private int ATTEMPT_ID = 1; // Incrementing counter for scheduling attempts
-
- // HELPER METHODS
@Before
public void setUp() throws IOException {
scheduler = new FairScheduler();
conf = createConfiguration();
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
- conf.setInt(FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
- 1024);
- conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 10240);
- // All tests assume only one assignment per node update
- conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
resourceManager = new ResourceManager();
resourceManager.init(conf);
@@ -198,107 +163,6 @@ public class TestFairScheduler {
}
}
- private Configuration createConfiguration() {
- Configuration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
- ResourceScheduler.class);
- return conf;
- }
-
- private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
- ApplicationId appIdImpl = ApplicationId.newInstance(0, appId);
- ApplicationAttemptId attId =
- ApplicationAttemptId.newInstance(appIdImpl, attemptId);
- return attId;
- }
-
- private ResourceRequest createResourceRequest(int memory, String host,
- int priority, int numContainers, boolean relaxLocality) {
- return createResourceRequest(memory, 1, host, priority, numContainers,
- relaxLocality);
- }
-
- private ResourceRequest createResourceRequest(int memory, int vcores, String host,
- int priority, int numContainers, boolean relaxLocality) {
- ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
- request.setCapability(BuilderUtils.newResource(memory, vcores));
- request.setResourceName(host);
- request.setNumContainers(numContainers);
- Priority prio = recordFactory.newRecordInstance(Priority.class);
- prio.setPriority(priority);
- request.setPriority(prio);
- request.setRelaxLocality(relaxLocality);
- return request;
- }
-
- /**
- * Creates a single container priority-1 request and submits to
- * scheduler.
- */
- private ApplicationAttemptId createSchedulingRequest(int memory, String queueId,
- String userId) {
- return createSchedulingRequest(memory, queueId, userId, 1);
- }
-
- private ApplicationAttemptId createSchedulingRequest(int memory, int vcores,
- String queueId, String userId) {
- return createSchedulingRequest(memory, vcores, queueId, userId, 1);
- }
-
- private ApplicationAttemptId createSchedulingRequest(int memory, String queueId,
- String userId, int numContainers) {
- return createSchedulingRequest(memory, queueId, userId, numContainers, 1);
- }
-
- private ApplicationAttemptId createSchedulingRequest(int memory, int vcores,
- String queueId, String userId, int numContainers) {
- return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1);
- }
-
- private ApplicationAttemptId createSchedulingRequest(int memory, String queueId,
- String userId, int numContainers, int priority) {
- return createSchedulingRequest(memory, 1, queueId, userId, numContainers,
- priority);
- }
-
- private ApplicationAttemptId createSchedulingRequest(int memory, int vcores,
- String queueId, String userId, int numContainers, int priority) {
- ApplicationAttemptId id = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
- scheduler.addApplication(id.getApplicationId(), queueId, userId);
- // This conditional is for testAclSubmitApplication where app is rejected
- // and no app is added.
- if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
- scheduler.addApplicationAttempt(id, false);
- }
- List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
- ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY,
- priority, numContainers, true);
- ask.add(request);
- scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null);
- return id;
- }
-
- private void createSchedulingRequestExistingApplication(int memory, int priority,
- ApplicationAttemptId attId) {
- ResourceRequest request = createResourceRequest(memory, ResourceRequest.ANY,
- priority, 1, true);
- createSchedulingRequestExistingApplication(request, attId);
- }
-
- private void createSchedulingRequestExistingApplication(int memory, int vcores,
- int priority, ApplicationAttemptId attId) {
- ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY,
- priority, 1, true);
- createSchedulingRequestExistingApplication(request, attId);
- }
-
- private void createSchedulingRequestExistingApplication(ResourceRequest request,
- ApplicationAttemptId attId) {
- List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
- ask.add(request);
- scheduler.allocate(attId, ask, new ArrayList<ContainerId>(), null, null);
- }
-
// TESTS
@Test(timeout=2000)
@@ -1165,13 +1029,13 @@ public class TestFairScheduler {
@Test (timeout = 5000)
/**
- * Make sure containers are chosen to be preempted in the correct order. Right
- * now this means decreasing order of priority.
+ * Make sure containers are chosen to be preempted in the correct order.
*/
public void testChoiceOfPreemptedContainers() throws Exception {
conf.setLong(FairSchedulerConfiguration.PREEMPTION_INTERVAL, 5000);
- conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000);
+ conf.setLong(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 10000);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE);
+ conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
MockClock clock = new MockClock();
scheduler.setClock(clock);
@@ -1188,7 +1052,7 @@ public class TestFairScheduler {
out.println("<queue name=\"queueC\">");
out.println("<weight>.25</weight>");
out.println("</queue>");
- out.println("<queue name=\"queueD\">");
+ out.println("<queue name=\"default\">");
out.println("<weight>.25</weight>");
out.println("</queue>");
out.println("</allocations>");
@@ -1196,133 +1060,132 @@ public class TestFairScheduler {
scheduler.reinitialize(conf, resourceManager.getRMContext());
- // Create four nodes
+ // Create two nodes
RMNode node1 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 1,
+ MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 1,
"127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.handle(nodeEvent1);
RMNode node2 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 2,
+ MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 2,
"127.0.0.2");
NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
scheduler.handle(nodeEvent2);
- RMNode node3 =
- MockNodes.newNodeInfo(1, Resources.createResource(2 * 1024, 2), 3,
- "127.0.0.3");
- NodeAddedSchedulerEvent nodeEvent3 = new NodeAddedSchedulerEvent(node3);
- scheduler.handle(nodeEvent3);
-
-
- // Queue A and B each request three containers
+ // Queue A and B each request two applications
ApplicationAttemptId app1 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 1);
+ createSchedulingRequest(1 * 1024, 1, "queueA", "user1", 1, 1);
+ createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app1);
ApplicationAttemptId app2 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 2);
- ApplicationAttemptId app3 =
- createSchedulingRequest(1 * 1024, "queueA", "user1", 1, 3);
+ createSchedulingRequest(1 * 1024, 1, "queueA", "user1", 1, 3);
+ createSchedulingRequestExistingApplication(1 * 1024, 1, 4, app2);
+ ApplicationAttemptId app3 =
+ createSchedulingRequest(1 * 1024, 1, "queueB", "user1", 1, 1);
+ createSchedulingRequestExistingApplication(1 * 1024, 1, 2, app3);
ApplicationAttemptId app4 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 1);
- ApplicationAttemptId app5 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 2);
- ApplicationAttemptId app6 =
- createSchedulingRequest(1 * 1024, "queueB", "user1", 1, 3);
+ createSchedulingRequest(1 * 1024, 1, "queueB", "user1", 1, 3);
+ createSchedulingRequestExistingApplication(1 * 1024, 1, 4, app4);
scheduler.update();
+ scheduler.getQueueManager().getLeafQueue("queueA", true)
+ .setPolicy(SchedulingPolicy.parse("fifo"));
+ scheduler.getQueueManager().getLeafQueue("queueB", true)
+ .setPolicy(SchedulingPolicy.parse("fair"));
+
// Sufficient node check-ins to fully schedule containers
- for (int i = 0; i < 2; i++) {
- NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
+ NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
+ NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
+ for (int i = 0; i < 4; i++) {
scheduler.handle(nodeUpdate1);
-
- NodeUpdateSchedulerEvent nodeUpdate2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(nodeUpdate2);
-
- NodeUpdateSchedulerEvent nodeUpdate3 = new NodeUpdateSchedulerEvent(node3);
- scheduler.handle(nodeUpdate3);
}
- assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app3).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app5).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app6).getLiveContainers().size());
-
- // Now new requests arrive from queues C and D
- ApplicationAttemptId app7 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 1);
- ApplicationAttemptId app8 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 2);
- ApplicationAttemptId app9 =
- createSchedulingRequest(1 * 1024, "queueC", "user1", 1, 3);
-
- ApplicationAttemptId app10 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 1);
- ApplicationAttemptId app11 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 2);
- ApplicationAttemptId app12 =
- createSchedulingRequest(1 * 1024, "queueD", "user1", 1, 3);
-
- scheduler.update();
-
- // We should be able to claw back one container from A and B each.
- // Make sure it is lowest priority container.
- scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
- Resources.createResource(2 * 1024));
- assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app5).getLiveContainers().size());
-
- // First verify we are adding containers to preemption list for the application
- assertTrue(!Collections.disjoint(scheduler.getSchedulerApp(app3).getLiveContainers(),
- scheduler.getSchedulerApp(app3).getPreemptionContainers()));
- assertTrue(!Collections.disjoint(scheduler.getSchedulerApp(app6).getLiveContainers(),
- scheduler.getSchedulerApp(app6).getPreemptionContainers()));
+ assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(2, scheduler.getSchedulerApp(app2).getLiveContainers().size());
+ assertEquals(2, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+ assertEquals(2, scheduler.getSchedulerApp(app4).getLiveContainers().size());
+
+ // Now new requests arrive from queueC and default
+ createSchedulingRequest(1 * 1024, 1, "queueC", "user1", 1, 1);
+ createSchedulingRequest(1 * 1024, 1, "queueC", "user1", 1, 1);
+ createSchedulingRequest(1 * 1024, 1, "default", "user1", 1, 1);
+ createSchedulingRequest(1 * 1024, 1, "default", "user1", 1, 1);
+ scheduler.update();
+
+ // We should be able to claw back one container from queueA and queueB each.
+ scheduler.preemptResources(Resources.createResource(2 * 1024));
+ assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(2, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+
+ // First verify we are adding containers to preemption list for the app.
+ // For queueA (fifo), app2 is selected.
+ // For queueB (fair), app4 is selected.
+ assertTrue("App2 should have container to be preempted",
+ !Collections.disjoint(
+ scheduler.getSchedulerApp(app2).getLiveContainers(),
+ scheduler.getSchedulerApp(app2).getPreemptionContainers()));
+ assertTrue("App4 should have container to be preempted",
+ !Collections.disjoint(
+ scheduler.getSchedulerApp(app2).getLiveContainers(),
+ scheduler.getSchedulerApp(app2).getPreemptionContainers()));
// Pretend 15 seconds have passed
clock.tick(15);
// Trigger a kill by insisting we want containers back
- scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
- Resources.createResource(2 * 1024));
+ scheduler.preemptResources(Resources.createResource(2 * 1024));
// At this point the containers should have been killed (since we are not simulating AM)
- assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
+ // Inside each app, containers are sorted according to their priorities.
+ // Containers with priority 4 are preempted for app2 and app4.
+ Set<RMContainer> set = new HashSet<RMContainer>();
+ for (RMContainer container :
+ scheduler.getSchedulerApp(app2).getLiveContainers()) {
+ if (container.getAllocatedPriority().getPriority() == 4) {
+ set.add(container);
+ }
+ }
+ for (RMContainer container :
+ scheduler.getSchedulerApp(app4).getLiveContainers()) {
+ if (container.getAllocatedPriority().getPriority() == 4) {
+ set.add(container);
+ }
+ }
+ assertTrue("Containers with priority=4 in app2 and app4 should be " +
+ "preempted.", set.isEmpty());
// Trigger a kill by insisting we want containers back
- scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
- Resources.createResource(2 * 1024));
+ scheduler.preemptResources(Resources.createResource(2 * 1024));
// Pretend 15 seconds have passed
clock.tick(15);
// We should be able to claw back another container from A and B each.
- // Make sure it is lowest priority container.
- scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
- Resources.createResource(2 * 1024));
-
- assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ // For queueA (fifo), continue preempting from app2.
+ // For queueB (fair), even app4 has a lowest priority container with p=4, it
+ // still preempts from app3 as app3 is most over fair share.
+ scheduler.preemptResources(Resources.createResource(2 * 1024));
+
+ assertEquals(2, scheduler.getSchedulerApp(app1).getLiveContainers().size());
assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app3).getLiveContainers().size());
assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app5).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size());
// Now A and B are below fair share, so preemption shouldn't do anything
- scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
- Resources.createResource(2 * 1024));
- assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size());
- assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app5).getLiveContainers().size());
- assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size());
+ scheduler.preemptResources(Resources.createResource(2 * 1024));
+ assertTrue("App1 should have no container to be preempted",
+ scheduler.getSchedulerApp(app1).getPreemptionContainers().isEmpty());
+ assertTrue("App2 should have no container to be preempted",
+ scheduler.getSchedulerApp(app2).getPreemptionContainers().isEmpty());
+ assertTrue("App3 should have no container to be preempted",
+ scheduler.getSchedulerApp(app3).getPreemptionContainers().isEmpty());
+ assertTrue("App4 should have no container to be preempted",
+ scheduler.getSchedulerApp(app4).getPreemptionContainers().isEmpty());
}
@Test (timeout = 5000)
@@ -1455,7 +1318,7 @@ public class TestFairScheduler {
assertEquals(
1536, scheduler.resToPreempt(schedD, clock.getTime()).getMemory());
}
-
+
@Test (timeout = 5000)
public void testMultipleContainersWaitingForReservation() throws IOException {
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -2560,6 +2423,35 @@ public class TestFairScheduler {
assertEquals(2, defaultQueue.getRunnableAppSchedulables().size());
}
+ @Test
+ public void testDefaultRuleInitializesProperlyWhenPolicyNotConfigured()
+ throws IOException {
+ // This test verifies if default rule in queue placement policy
+ // initializes properly when policy is not configured and
+ // undeclared pools is not allowed.
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
+
+ // Create an alloc file with no queue placement policy
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ List<QueuePlacementRule> rules = scheduler.allocConf.placementPolicy
+ .getRules();
+
+ for (QueuePlacementRule rule : rules) {
+ if (rule instanceof Default) {
+ Default defaultRule = (Default) rule;
+ assertNotNull(defaultRule.defaultQueueName);
+ }
+ }
+ }
+
@SuppressWarnings("resource")
@Test
public void testBlacklistNodes() throws Exception {
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java Thu May 29 22:27:25 2014
@@ -134,6 +134,22 @@ public class TestQueuePlacementPolicy {
}
@Test
+ public void testDefaultRuleWithQueueAttribute() throws Exception {
+ // This test covers the use case where we would like default rule
+ // to point to a different queue by default rather than root.default
+ configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue");
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='default' queue='root.someDefaultQueue'/>");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.someDefaultQueue",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
+ @Test
public void testNestedUserQueueParsingErrors() {
// No nested rule specified in hierarchical user queue
StringBuffer sb = new StringBuffer();
@@ -311,6 +327,25 @@ public class TestQueuePlacementPolicy {
policy.assignAppToQueue("root.parent2", "user2"));
}
+ @Test
+ public void testNestedUserQueueDefaultRule() throws Exception {
+ // This test covers the use case where we would like user queues to be
+ // created under a default parent queue
+ configuredQueues.get(FSQueueType.PARENT).add("root.parentq");
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='default' queue='root.parentq'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.parentq.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
private QueuePlacementPolicy parse(String str) throws Exception {
// Read and parse the allocations file.
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Thu May 29 22:27:25 2014
@@ -25,6 +25,7 @@ import java.net.UnknownHostException;
import java.util.Collection;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -56,8 +57,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -69,6 +69,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
+import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
+import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -719,6 +721,7 @@ public class MiniYARNCluster extends Com
if (appHistoryServer != null) {
appHistoryServer.stop();
}
+ AHSWebApp.resetInstance();
super.serviceStop();
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Hadoop MapReduce Next Generation - Capacity Scheduler
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* {Purpose}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Hadoop MapReduce Next Generation - Fair Scheduler
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* {Purpose}
@@ -156,6 +154,12 @@ Properties that can be placed in yarn-si
* Whether to use preemption. Note that preemption is experimental in the current
version. Defaults to false.
+ * <<<yarn.scheduler.fair.preemption.cluster-utilization-threshold>>>
+
+ * The utilization threshold after which preemption kicks in. The
+ utilization is computed as the maximum ratio of usage to capacity among
+ all resources. Defaults to 0.8f.
+
* <<<yarn.scheduler.fair.sizebasedweight>>>
* Whether to assign shares to individual apps based on their size, rather than
@@ -310,7 +314,8 @@ Allocation file format
to âparentâ or by configuring at least one leaf under that queue which makes it a parent.
See example allocation for a sample use case.
- * default: the app is placed into the queue named "default".
+ * default: the app is placed into the queue specified in the âqueueâ attribute of the
+ default rule. If âqueueâ attribute is not specified, the app is placed into âroot.defaultâ queue.
* reject: the app is rejected.
@@ -348,7 +353,7 @@ Allocation file format
<rule name=ânestedUserQueueâ>
<rule name=âsecondaryGroupExistingQueueâ create=âfalseâ />
</rule>
- <rule name="default" />
+ <rule name="default" queue=âsample_queueâ />
</queuePlacementPolicy>
</allocations>
---
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManager.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
NodeManager Overview.
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0|toDepth=2}
* Overview
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
NodeManager REST API's.
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0|toDepth=2}
* Overview
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerHA.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
ResourceManager High Availability
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* Introduction
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
ResourceManager REST API's.
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0|toDepth=2}
* Overview
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
YARN Timeline Server
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0|toDepth=3}
* Overview
@@ -152,7 +150,7 @@ YARN Timeline Server
<property>
<description>Store class name for timeline store.</description>
<name>yarn.timeline-service.store-class</name>
- <value>org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore</value>
+ <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
</property>
<property>
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WebServicesIntro.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Hadoop YARN - Introduction to the web services REST API's.
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* Overview
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm Thu May 29 22:27:25 2014
@@ -19,8 +19,6 @@
Hadoop MapReduce Next Generation - Writing YARN Applications
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* Purpose
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm?rev=1598435&r1=1598434&r2=1598435&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm Thu May 29 22:27:25 2014
@@ -18,8 +18,6 @@
Yarn Commands
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* Overview