You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ar...@apache.org on 2014/08/23 09:49:45 UTC
svn commit: r1619980 - in
/hadoop/common/branches/HDFS-6581/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/
hadoop-yarn/hadoop-yarn-common/src/test/java/org/apa...
Author: arp
Date: Sat Aug 23 07:49:41 2014
New Revision: 1619980
URL: http://svn.apache.org/r1619980
Log:
Merging r1619458 through r1619979 from trunk to branch HDFS-6581
Removed:
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApplicationClassLoader.java
Modified:
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/CHANGES.txt Sat Aug 23 07:49:41 2014
@@ -18,6 +18,8 @@ Trunk - Unreleased
YARN-2216 TestRMApplicationHistoryWriter sometimes fails in trunk.
(Zhijie Shen via xgong)
+ YARN-2436. [post-HADOOP-9902] yarn application help doesn't work (aw)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -53,6 +55,9 @@ Release 2.6.0 - UNRELEASED
YARN-2174. Enable HTTPs for the writer REST API of TimelineServer.
(Zhijie Shen via jianhe)
+ YARN-2393. FairScheduler: Add the notion of steady fair share.
+ (Wei Yan via kasha)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
@@ -229,6 +234,9 @@ Release 2.6.0 - UNRELEASED
YARN-2424. LCE should support non-cgroups, non-secure mode (Chris Douglas
via aw)
+ YARN-2434. RM should not recover containers from previously failed attempt
+ when AM restart is not enabled (Jian He via jlowe)
+
Release 2.5.0 - 2014-08-11
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/bin/yarn Sat Aug 23 07:49:41 2014
@@ -73,6 +73,7 @@ case "${COMMAND}" in
application|applicationattempt|container)
CLASS=org.apache.hadoop.yarn.client.cli.ApplicationCLI
YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
+ set -- "${COMMAND}" "$@"
;;
classpath)
hadoop_finalize
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml Sat Aug 23 07:49:41 2014
@@ -344,4 +344,11 @@
<Class name="org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider"/>
<Bug pattern="DC_DOUBLECHECK" />
</Match>
+
+ <!-- ApplicationClassLoader is deprecated and moved to hadoop-common; ignore
+ warning on the identical name as it should be removed later -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.util.ApplicationClassLoader"/>
+ <Bug pattern="NM_SAME_SIMPLE_NAME_AS_SUPERCLASS"/>
+ </Match>
</FindBugsFilter>
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java Sat Aug 23 07:49:41 2014
@@ -18,180 +18,30 @@
package org.apache.hadoop.yarn.util;
-import java.io.File;
-import java.io.FilenameFilter;
import java.net.MalformedURLException;
import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
import java.util.List;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Splitter;
-
/**
- * A {@link URLClassLoader} for YARN application isolation. Classes from
- * the application JARs are loaded in preference to the parent loader.
+ * This type has been deprecated in favor of
+ * {@link org.apache.hadoop.util.ApplicationClassLoader}. All new uses of
+ * ApplicationClassLoader should use that type instead.
*/
@Public
@Unstable
-public class ApplicationClassLoader extends URLClassLoader {
-
- private static final Log LOG =
- LogFactory.getLog(ApplicationClassLoader.class.getName());
-
- private static final FilenameFilter JAR_FILENAME_FILTER =
- new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(".jar") || name.endsWith(".JAR");
- }
- };
-
- private ClassLoader parent;
- private List<String> systemClasses;
-
+@Deprecated
+public class ApplicationClassLoader extends
+ org.apache.hadoop.util.ApplicationClassLoader {
public ApplicationClassLoader(URL[] urls, ClassLoader parent,
List<String> systemClasses) {
- super(urls, parent);
- this.parent = parent;
- if (parent == null) {
- throw new IllegalArgumentException("No parent classloader!");
- }
- this.systemClasses = systemClasses;
+ super(urls, parent, systemClasses);
}
-
+
public ApplicationClassLoader(String classpath, ClassLoader parent,
List<String> systemClasses) throws MalformedURLException {
- this(constructUrlsFromClasspath(classpath), parent, systemClasses);
- }
-
- @VisibleForTesting
- static URL[] constructUrlsFromClasspath(String classpath)
- throws MalformedURLException {
- List<URL> urls = new ArrayList<URL>();
- for (String element : Splitter.on(File.pathSeparator).split(classpath)) {
- if (element.endsWith("/*")) {
- String dir = element.substring(0, element.length() - 1);
- File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
- if (files != null) {
- for (File file : files) {
- urls.add(file.toURI().toURL());
- }
- }
- } else {
- File file = new File(element);
- if (file.exists()) {
- urls.add(new File(element).toURI().toURL());
- }
- }
- }
- return urls.toArray(new URL[urls.size()]);
- }
-
- @Override
- public URL getResource(String name) {
- URL url = null;
-
- if (!isSystemClass(name, systemClasses)) {
- url= findResource(name);
- if (url == null && name.startsWith("/")) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Remove leading / off " + name);
- }
- url= findResource(name.substring(1));
- }
- }
-
- if (url == null) {
- url= parent.getResource(name);
- }
-
- if (url != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("getResource("+name+")=" + url);
- }
- }
-
- return url;
- }
-
- @Override
- public Class<?> loadClass(String name) throws ClassNotFoundException {
- return this.loadClass(name, false);
- }
-
- @Override
- protected synchronized Class<?> loadClass(String name, boolean resolve)
- throws ClassNotFoundException {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Loading class: " + name);
- }
-
- Class<?> c = findLoadedClass(name);
- ClassNotFoundException ex = null;
-
- if (c == null && !isSystemClass(name, systemClasses)) {
- // Try to load class from this classloader's URLs. Note that this is like
- // the servlet spec, not the usual Java 2 behaviour where we ask the
- // parent to attempt to load first.
- try {
- c = findClass(name);
- if (LOG.isDebugEnabled() && c != null) {
- LOG.debug("Loaded class: " + name + " ");
- }
- } catch (ClassNotFoundException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(e);
- }
- ex = e;
- }
- }
-
- if (c == null) { // try parent
- c = parent.loadClass(name);
- if (LOG.isDebugEnabled() && c != null) {
- LOG.debug("Loaded class from parent: " + name + " ");
- }
- }
-
- if (c == null) {
- throw ex != null ? ex : new ClassNotFoundException(name);
- }
-
- if (resolve) {
- resolveClass(c);
- }
-
- return c;
- }
-
- @VisibleForTesting
- public static boolean isSystemClass(String name, List<String> systemClasses) {
- if (systemClasses != null) {
- String canonicalName = name.replace('/', '.');
- while (canonicalName.startsWith(".")) {
- canonicalName=canonicalName.substring(1);
- }
- for (String c : systemClasses) {
- boolean result = true;
- if (c.startsWith("-")) {
- c = c.substring(1);
- result = false;
- }
- if (c.endsWith(".") && canonicalName.startsWith(c)) {
- return result;
- } else if (canonicalName.equals(c)) {
- return result;
- }
- }
- }
- return false;
+ super(classpath, parent, systemClasses);
}
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java Sat Aug 23 07:49:41 2014
@@ -273,6 +273,19 @@ public abstract class AbstractYarnSchedu
SchedulerApplicationAttempt schedulerAttempt =
schedulerApp.getCurrentAppAttempt();
+ if (!rmApp.getApplicationSubmissionContext()
+ .getKeepContainersAcrossApplicationAttempts()) {
+ // Do not recover containers for stopped attempt or previous attempt.
+ if (schedulerAttempt.isStopped()
+ || !schedulerAttempt.getApplicationAttemptId().equals(
+ container.getContainerId().getApplicationAttemptId())) {
+ LOG.info("Skip recovering container " + container
+ + " for already stopped attempt.");
+ killOrphanContainerOnNode(nm, container);
+ continue;
+ }
+ }
+
// create container
RMContainer rmContainer = recoverAndCreateContainer(container, nm);
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java Sat Aug 23 07:49:41 2014
@@ -718,12 +718,6 @@ public class FSAppAttempt extends Schedu
}
@Override
- public boolean isActive() {
- return true;
- }
-
-
- @Override
public void updateDemand() {
demand = Resources.createResource(0);
// Demand is current consumption plus outstanding requests
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java Sat Aug 23 07:49:41 2014
@@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@@ -68,6 +67,16 @@ public class FSParentQueue extends FSQue
}
}
+ public void recomputeSteadyShares() {
+ policy.computeSteadyShares(childQueues, getSteadyFairShare());
+ for (FSQueue childQueue : childQueues) {
+ childQueue.getMetrics().setSteadyFairShare(childQueue.getSteadyFairShare());
+ if (childQueue instanceof FSParentQueue) {
+ ((FSParentQueue) childQueue).recomputeSteadyShares();
+ }
+ }
+ }
+
@Override
public Resource getDemand() {
return demand;
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java Sat Aug 23 07:49:41 2014
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.util.resou
@Unstable
public abstract class FSQueue implements Queue, Schedulable {
private Resource fairShare = Resources.createResource(0, 0);
+ private Resource steadyFairShare = Resources.createResource(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final FSQueueMetrics metrics;
@@ -151,7 +152,17 @@ public abstract class FSQueue implements
this.fairShare = fairShare;
metrics.setFairShare(fairShare);
}
-
+
+ /** Get the steady fair share assigned to this Schedulable. */
+ public Resource getSteadyFairShare() {
+ return steadyFairShare;
+ }
+
+ public void setSteadyFairShare(Resource steadyFairShare) {
+ this.steadyFairShare = steadyFairShare;
+ metrics.setSteadyFairShare(steadyFairShare);
+ }
+
public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
return scheduler.getAllocationConfiguration().hasAccess(name, acl, user);
}
@@ -161,7 +172,7 @@ public abstract class FSQueue implements
* queue's current share
*/
public abstract void recomputeShares();
-
+
/**
* Gets the children of this queue, if any.
*/
@@ -194,7 +205,9 @@ public abstract class FSQueue implements
return true;
}
- @Override
+ /**
+ * Returns true if queue has at least one app running.
+ */
public boolean isActive() {
return getNumRunnableApps() > 0;
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueueMetrics.java Sat Aug 23 07:49:41 2014
@@ -33,6 +33,8 @@ public class FSQueueMetrics extends Queu
@Metric("Fair share of memory in MB") MutableGaugeInt fairShareMB;
@Metric("Fair share of CPU in vcores") MutableGaugeInt fairShareVCores;
+ @Metric("Steady fair share of memory in MB") MutableGaugeInt steadyFairShareMB;
+ @Metric("Steady fair share of CPU in vcores") MutableGaugeInt steadyFairShareVCores;
@Metric("Minimum share of memory in MB") MutableGaugeInt minShareMB;
@Metric("Minimum share of CPU in vcores") MutableGaugeInt minShareVCores;
@Metric("Maximum share of memory in MB") MutableGaugeInt maxShareMB;
@@ -55,7 +57,20 @@ public class FSQueueMetrics extends Queu
public int getFairShareVirtualCores() {
return fairShareVCores.value();
}
-
+
+ public void setSteadyFairShare(Resource resource) {
+ steadyFairShareMB.set(resource.getMemory());
+ steadyFairShareVCores.set(resource.getVirtualCores());
+ }
+
+ public int getSteadyFairShareMB() {
+ return steadyFairShareMB.value();
+ }
+
+ public int getSteadyFairShareVCores() {
+ return steadyFairShareVCores.value();
+ }
+
public void setMinShare(Resource resource) {
minShareMB.set(resource.getMemory());
minShareVCores.set(resource.getVirtualCores());
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Sat Aug 23 07:49:41 2014
@@ -851,6 +851,8 @@ public class FairScheduler extends
Resources.addTo(clusterResource, node.getTotalCapability());
updateRootQueueMetrics();
+ queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
+ queueMgr.getRootQueue().recomputeSteadyShares();
LOG.info("Added node " + node.getNodeAddress() +
" cluster capacity: " + clusterResource);
}
@@ -885,6 +887,8 @@ public class FairScheduler extends
}
nodes.remove(rmNode.getNodeID());
+ queueMgr.getRootQueue().setSteadyFairShare(clusterResource);
+ queueMgr.getRootQueue().recomputeSteadyShares();
LOG.info("Removed node " + rmNode.getNodeAddress() +
" cluster capacity: " + clusterResource);
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java Sat Aug 23 07:49:41 2014
@@ -118,6 +118,11 @@ public class QueueManager {
if (queue == null && create) {
// if the queue doesn't exist,create it and return
queue = createQueue(name, queueType);
+
+ // Update steady fair share for all queues
+ if (queue != null) {
+ rootQueue.recomputeSteadyShares();
+ }
}
return queue;
}
@@ -190,7 +195,7 @@ public class QueueManager {
parent = newParent;
}
}
-
+
return parent;
}
@@ -376,5 +381,8 @@ public class QueueManager {
+ queue.getName(), ex);
}
}
+
+ // Update steady fair shares for all queues
+ rootQueue.recomputeSteadyShares();
}
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Schedulable.java Sat Aug 23 07:49:41 2014
@@ -24,7 +24,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.util.resource.Resources;
/**
* A Schedulable represents an entity that can be scheduled such as an
@@ -102,10 +101,4 @@ public interface Schedulable {
/** Assign a fair share to this Schedulable. */
public void setFairShare(Resource fairShare);
-
- /**
- * Returns true if queue has atleast one app running. Always returns true for
- * AppSchedulables.
- */
- public boolean isActive();
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingPolicy.java Sat Aug 23 07:49:41 2014
@@ -17,10 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import java.util.Collection;
-import java.util.Comparator;
-import java.util.concurrent.ConcurrentHashMap;
-
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.util.ReflectionUtils;
@@ -29,6 +25,10 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.concurrent.ConcurrentHashMap;
+
@Public
@Evolving
public abstract class SchedulingPolicy {
@@ -131,8 +131,10 @@ public abstract class SchedulingPolicy {
public abstract Comparator<Schedulable> getComparator();
/**
- * Computes and updates the shares of {@link Schedulable}s as per the
- * {@link SchedulingPolicy}, to be used later at schedule time.
+ * Computes and updates the shares of {@link Schedulable}s as per
+ * the {@link SchedulingPolicy}, to be used later for scheduling decisions.
+ * The shares computed are instantaneous and only consider queues with
+ * running applications.
*
* @param schedulables {@link Schedulable}s whose shares are to be updated
* @param totalResources Total {@link Resource}s in the cluster
@@ -141,6 +143,19 @@ public abstract class SchedulingPolicy {
Collection<? extends Schedulable> schedulables, Resource totalResources);
/**
+ * Computes and updates the steady shares of {@link FSQueue}s as per the
+ * {@link SchedulingPolicy}. The steady share does not differentiate
+ * between queues with and without running applications under them. The
+ * steady share is not used for scheduling, it is displayed on the Web UI
+ * for better visibility.
+ *
+ * @param queues {@link FSQueue}s whose shares are to be updated
+ * @param totalResources Total {@link Resource}s in the cluster
+ */
+ public abstract void computeSteadyShares(
+ Collection<? extends FSQueue> queues, Resource totalResources);
+
+ /**
* Check if the resource usage is over the fair share under this policy
*
* @param usage {@link Resource} the resource usage
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/ComputeFairShares.java Sat Aug 23 07:49:41 2014
@@ -22,6 +22,7 @@ import java.util.Collection;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
/**
@@ -49,14 +50,29 @@ public class ComputeFairShares {
ResourceType type) {
Collection<Schedulable> activeSchedulables = new ArrayList<Schedulable>();
for (Schedulable sched : schedulables) {
- if (sched.isActive()) {
- activeSchedulables.add(sched);
- } else {
+ if ((sched instanceof FSQueue) && !((FSQueue) sched).isActive()) {
setResourceValue(0, sched.getFairShare(), type);
+ } else {
+ activeSchedulables.add(sched);
}
}
- computeSharesInternal(activeSchedulables, totalResources, type);
+ computeSharesInternal(activeSchedulables, totalResources, type, false);
+ }
+
+ /**
+ * Compute the steady fair share of the given queues. The steady fair
+ * share is an allocation of shares considering all queues, i.e.,
+ * active and inactive.
+ *
+ * @param queues
+ * @param totalResources
+ * @param type
+ */
+ public static void computeSteadyShares(
+ Collection<? extends FSQueue> queues, Resource totalResources,
+ ResourceType type) {
+ computeSharesInternal(queues, totalResources, type, true);
}
/**
@@ -102,7 +118,7 @@ public class ComputeFairShares {
*/
private static void computeSharesInternal(
Collection<? extends Schedulable> schedulables, Resource totalResources,
- ResourceType type) {
+ ResourceType type, boolean isSteadyShare) {
if (schedulables.isEmpty()) {
return;
}
@@ -145,7 +161,13 @@ public class ComputeFairShares {
}
// Set the fair shares based on the value of R we've converged to
for (Schedulable sched : schedulables) {
- setResourceValue(computeShare(sched, right, type), sched.getFairShare(), type);
+ if (isSteadyShare) {
+ setResourceValue(computeShare(sched, right, type),
+ ((FSQueue) sched).getSteadyFairShare(), type);
+ } else {
+ setResourceValue(
+ computeShare(sched, right, type), sched.getFairShare(), type);
+ }
}
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java Sat Aug 23 07:49:41 2014
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -68,6 +69,14 @@ public class DominantResourceFairnessPol
ComputeFairShares.computeShares(schedulables, totalResources, type);
}
}
+
+ @Override
+ public void computeSteadyShares(Collection<? extends FSQueue> queues,
+ Resource totalResources) {
+ for (ResourceType type : ResourceType.values()) {
+ ComputeFairShares.computeSteadyShares(queues, totalResources, type);
+ }
+ }
@Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java Sat Aug 23 07:49:41 2014
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -120,6 +121,13 @@ public class FairSharePolicy extends Sch
}
@Override
+ public void computeSteadyShares(Collection<? extends FSQueue> queues,
+ Resource totalResources) {
+ ComputeFairShares.computeSteadyShares(queues, totalResources,
+ ResourceType.MEMORY);
+ }
+
+ @Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
return Resources.greaterThan(RESOURCE_CALCULATOR, null, usage, fairShare);
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java Sat Aug 23 07:49:41 2014
@@ -24,6 +24,7 @@ import java.util.Comparator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.Schedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.SchedulingPolicy;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -88,6 +89,13 @@ public class FifoPolicy extends Scheduli
}
@Override
+ public void computeSteadyShares(Collection<? extends FSQueue> queues,
+ Resource totalResources) {
+ // Nothing needs to do, as leaf queue doesn't have to calculate steady
+ // fair shares for applications.
+ }
+
+ @Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
throw new UnsupportedOperationException(
"FifoPolicy doesn't support checkIfUsageOverFairshare operation, " +
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java Sat Aug 23 07:49:41 2014
@@ -513,6 +513,19 @@ public class TestWorkPreservingRMRestart
// just-recovered containers.
assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
+
+ rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
+
+ MockNM nm2 =
+ new MockNM("127.1.1.1:4321", 8192, rm2.getResourceTrackerService());
+ NMContainerStatus previousAttemptContainer =
+ TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 4,
+ ContainerState.RUNNING);
+ nm2.registerNode(Arrays.asList(previousAttemptContainer), null);
+ // Wait for RM to settle down on recovering containers;
+ Thread.sleep(3000);
+ // check containers from previous failed attempt should not be recovered.
+ assertNull(scheduler.getRMContainer(previousAttemptContainer.getContainerId()));
}
// Apps already completed before RM restart. Restarted RM scheduler should not
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FakeSchedulable.java Sat Aug 23 07:49:41 2014
@@ -101,11 +101,6 @@ public class FakeSchedulable implements
}
@Override
- public boolean isActive() {
- return true;
- }
-
- @Override
public Resource getDemand() {
return null;
}
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Sat Aug 23 07:49:41 2014
@@ -292,14 +292,19 @@ public class TestFairScheduler extends F
createSchedulingRequest(10 * 1024, "root.default", "user1");
scheduler.update();
+ scheduler.getQueueManager().getRootQueue()
+ .setSteadyFairShare(scheduler.getClusterResource());
+ scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();
Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
assertEquals(3, queues.size());
- // Divided three ways - betwen the two queues and the default queue
+ // Divided three ways - between the two queues and the default queue
for (FSLeafQueue p : queues) {
assertEquals(3414, p.getFairShare().getMemory());
assertEquals(3414, p.getMetrics().getFairShareMB());
+ assertEquals(3414, p.getSteadyFairShare().getMemory());
+ assertEquals(3414, p.getMetrics().getSteadyFairShareMB());
}
}
@@ -323,6 +328,9 @@ public class TestFairScheduler extends F
createSchedulingRequest(10 * 1024, "root.default", "user1");
scheduler.update();
+ scheduler.getQueueManager().getRootQueue()
+ .setSteadyFairShare(scheduler.getClusterResource());
+ scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();
QueueManager queueManager = scheduler.getQueueManager();
Collection<FSLeafQueue> queues = queueManager.getLeafQueues();
@@ -333,10 +341,16 @@ public class TestFairScheduler extends F
FSLeafQueue queue3 = queueManager.getLeafQueue("parent.queue3", true);
assertEquals(capacity / 2, queue1.getFairShare().getMemory());
assertEquals(capacity / 2, queue1.getMetrics().getFairShareMB());
+ assertEquals(capacity / 2, queue1.getSteadyFairShare().getMemory());
+ assertEquals(capacity / 2, queue1.getMetrics().getSteadyFairShareMB());
assertEquals(capacity / 4, queue2.getFairShare().getMemory());
assertEquals(capacity / 4, queue2.getMetrics().getFairShareMB());
+ assertEquals(capacity / 4, queue2.getSteadyFairShare().getMemory());
+ assertEquals(capacity / 4, queue2.getMetrics().getSteadyFairShareMB());
assertEquals(capacity / 4, queue3.getFairShare().getMemory());
assertEquals(capacity / 4, queue3.getMetrics().getFairShareMB());
+ assertEquals(capacity / 4, queue3.getSteadyFairShare().getMemory());
+ assertEquals(capacity / 4, queue3.getMetrics().getSteadyFairShareMB());
}
@Test
@@ -771,6 +785,9 @@ public class TestFairScheduler extends F
createSchedulingRequest(10 * 1024, "root.default", "user3");
scheduler.update();
+ scheduler.getQueueManager().getRootQueue()
+ .setSteadyFairShare(scheduler.getClusterResource());
+ scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();
Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
.getLeafQueues();
@@ -780,12 +797,128 @@ public class TestFairScheduler extends F
|| leaf.getName().equals("root.parentq.user2")) {
// assert that the fair share is 1/4th node1's capacity
assertEquals(capacity / 4, leaf.getFairShare().getMemory());
+ // assert that the steady fair share is 1/4th node1's capacity
+ assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemory());
// assert weights are equal for both the user queues
assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0);
}
}
}
-
+
+ @Test
+ public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
+ out.println("<queue name=\"root\">");
+ out.println(" <schedulingPolicy>drf</schedulingPolicy>");
+ out.println(" <queue name=\"child1\">");
+ out.println(" <weight>1</weight>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"child2\">");
+ out.println(" <weight>1</weight>");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // The steady fair share for all queues should be 0
+ QueueManager queueManager = scheduler.getQueueManager();
+ assertEquals(0, queueManager.getLeafQueue("child1", false)
+ .getSteadyFairShare().getMemory());
+ assertEquals(0, queueManager.getLeafQueue("child2", false)
+ .getSteadyFairShare().getMemory());
+
+ // Add one node
+ RMNode node1 =
+ MockNodes
+ .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ assertEquals(6144, scheduler.getClusterResource().getMemory());
+
+ // The steady fair shares for all queues should be updated
+ assertEquals(2048, queueManager.getLeafQueue("child1", false)
+ .getSteadyFairShare().getMemory());
+ assertEquals(2048, queueManager.getLeafQueue("child2", false)
+ .getSteadyFairShare().getMemory());
+
+ // Reload the allocation configuration file
+ out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
+ out.println("<queue name=\"root\">");
+ out.println(" <schedulingPolicy>drf</schedulingPolicy>");
+ out.println(" <queue name=\"child1\">");
+ out.println(" <weight>1</weight>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"child2\">");
+ out.println(" <weight>2</weight>");
+ out.println(" </queue>");
+ out.println(" <queue name=\"child3\">");
+ out.println(" <weight>2</weight>");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // The steady fair shares for all queues should be updated
+ assertEquals(1024, queueManager.getLeafQueue("child1", false)
+ .getSteadyFairShare().getMemory());
+ assertEquals(2048, queueManager.getLeafQueue("child2", false)
+ .getSteadyFairShare().getMemory());
+ assertEquals(2048, queueManager.getLeafQueue("child3", false)
+ .getSteadyFairShare().getMemory());
+
+ // Remove the node, steady fair shares should back to 0
+ NodeRemovedSchedulerEvent nodeEvent2 = new NodeRemovedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent2);
+ assertEquals(0, scheduler.getClusterResource().getMemory());
+ assertEquals(0, queueManager.getLeafQueue("child1", false)
+ .getSteadyFairShare().getMemory());
+ assertEquals(0, queueManager.getLeafQueue("child2", false)
+ .getSteadyFairShare().getMemory());
+ }
+
+ @Test
+ public void testSteadyFairShareWithQueueCreatedRuntime() throws Exception {
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
+ conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add one node
+ RMNode node1 =
+ MockNodes
+ .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ assertEquals(6144, scheduler.getClusterResource().getMemory());
+ assertEquals(6144, scheduler.getQueueManager().getRootQueue()
+ .getSteadyFairShare().getMemory());
+ assertEquals(6144, scheduler.getQueueManager()
+ .getLeafQueue("default", false).getSteadyFairShare().getMemory());
+
+ // Submit one application
+ ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1);
+ createApplicationWithAMResource(appAttemptId1, "default", "user1", null);
+ assertEquals(3072, scheduler.getQueueManager()
+ .getLeafQueue("default", false).getSteadyFairShare().getMemory());
+ assertEquals(3072, scheduler.getQueueManager()
+ .getLeafQueue("user1", false).getSteadyFairShare().getMemory());
+ }
+
/**
* Make allocation requests and ensure they are reflected in queue demand.
*/
@@ -873,7 +1006,7 @@ public class TestFairScheduler extends F
}
@Test
- public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,
+ public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,
AllocationConfigurationException, ParserConfigurationException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
Modified: hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java?rev=1619980&r1=1619979&r2=1619980&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java (original)
+++ hadoop/common/branches/HDFS-6581/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java Sat Aug 23 07:49:41 2014
@@ -109,13 +109,15 @@ public class TestFairSchedulerFairShare
for (FSLeafQueue leaf : leafQueues) {
if (leaf.getName().startsWith("root.parentA")) {
- assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity
- * 100, 0);
+ assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity,
+ 0);
} else if (leaf.getName().startsWith("root.parentB")) {
- assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity
- * 100, 0.1);
+ assertEquals(0, (double) leaf.getFairShare().getMemory() / nodeCapacity,
+ 0);
}
}
+
+ verifySteadyFairShareMemory(leafQueues, nodeCapacity);
}
@Test
@@ -135,14 +137,15 @@ public class TestFairSchedulerFairShare
100,
(double) scheduler.getQueueManager()
.getLeafQueue("root.parentA.childA1", false).getFairShare()
- .getMemory()
- / nodeCapacity * 100, 0.1);
+ .getMemory() / nodeCapacity * 100, 0.1);
assertEquals(
0,
(double) scheduler.getQueueManager()
.getLeafQueue("root.parentA.childA2", false).getFairShare()
- .getMemory()
- / nodeCapacity * 100, 0.1);
+ .getMemory() / nodeCapacity, 0.1);
+
+ verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+ nodeCapacity);
}
@Test
@@ -167,6 +170,9 @@ public class TestFairSchedulerFairShare
.getMemory()
/ nodeCapacity * 100, .9);
}
+
+ verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+ nodeCapacity);
}
@Test
@@ -206,6 +212,9 @@ public class TestFairSchedulerFairShare
.getLeafQueue("root.parentB.childB1", false).getFairShare()
.getMemory()
/ nodeCapacity * 100, .9);
+
+ verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+ nodeCapacity);
}
@Test
@@ -253,6 +262,9 @@ public class TestFairSchedulerFairShare
.getLeafQueue("root.parentA.childA2", false).getFairShare()
.getMemory()
/ nodeCapacity * 100, 0.1);
+
+ verifySteadyFairShareMemory(scheduler.getQueueManager().getLeafQueues(),
+ nodeCapacity);
}
@Test
@@ -304,5 +316,45 @@ public class TestFairSchedulerFairShare
.getLeafQueue("root.parentB.childB1", false).getFairShare()
.getVirtualCores()
/ nodeVCores * 100, .9);
+ Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
+ .getLeafQueues();
+
+ for (FSLeafQueue leaf : leafQueues) {
+ if (leaf.getName().startsWith("root.parentA")) {
+ assertEquals(0.2,
+ (double) leaf.getSteadyFairShare().getMemory() / nodeMem, 0.001);
+ assertEquals(0.2,
+ (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores,
+ 0.001);
+ } else if (leaf.getName().startsWith("root.parentB")) {
+ assertEquals(0.05,
+ (double) leaf.getSteadyFairShare().getMemory() / nodeMem, 0.001);
+ assertEquals(0.1,
+ (double) leaf.getSteadyFairShare().getVirtualCores() / nodeVCores,
+ 0.001);
+ }
+ }
+ }
+
+ /**
+ * Verify whether steady fair shares for all leaf queues still follow
+ * their weight, not related to active/inactive status.
+ *
+ * @param leafQueues
+ * @param nodeCapacity
+ */
+ private void verifySteadyFairShareMemory(Collection<FSLeafQueue> leafQueues,
+ int nodeCapacity) {
+ for (FSLeafQueue leaf : leafQueues) {
+ if (leaf.getName().startsWith("root.parentA")) {
+ assertEquals(0.2,
+ (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity,
+ 0.001);
+ } else if (leaf.getName().startsWith("root.parentB")) {
+ assertEquals(0.05,
+ (double) leaf.getSteadyFairShare().getMemory() / nodeCapacity,
+ 0.001);
+ }
+ }
}
}