You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/07/05 17:43:38 UTC
hadoop git commit: YARN-7556. Fair scheduler configuration should
allow resource types in the minResources and maxResources properties. (Daniel
Templeton and Szilard Nemeth via Haibo Chen)
Repository: hadoop
Updated Branches:
refs/heads/trunk 71df8c27c -> 172624702
YARN-7556. Fair scheduler configuration should allow resource types in the minResources and maxResources properties. (Daniel Templeton and Szilard Nemeth via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17262470
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17262470
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17262470
Branch: refs/heads/trunk
Commit: 17262470246232d0f0651d627a4961e55b1efe6a
Parents: 71df8c2
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Jul 5 10:42:39 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Jul 5 10:42:39 2018 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 17 +-
.../hadoop/yarn/api/records/Resource.java | 13 ++
.../api/records/impl/LightWeightResource.java | 23 ++-
.../scheduler/fair/ConfigurableResource.java | 69 +++++++-
.../fair/FairSchedulerConfiguration.java | 174 ++++++++++++++++---
.../allocation/AllocationFileQueueParser.java | 2 +-
.../fair/TestFairSchedulerConfiguration.java | 151 ++++++++++++----
.../src/site/markdown/FairScheduler.md | 6 +-
8 files changed, 385 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 5841361..5cc81e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -67,11 +67,6 @@
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
- <Method name="getLocalityStatistics" />
- <Bug pattern="EI_EXPOSE_REP" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
<Method name="incNumAllocatedContainers"/>
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
@@ -118,6 +113,18 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
+ <!-- Ignore exposed internal representations -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.api.records.Resource" />
+ <Method name="getResources" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
+ <Method name="getLocalityStatistics" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+
<!-- Object cast is based on the event type -->
<Match>
<Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" />
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 71a6b54..173d4c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
@@ -75,6 +76,18 @@ public abstract class Resource implements Comparable<Resource> {
@Private
public static final int VCORES_INDEX = 1;
+ /**
+ * Return a new {@link Resource} instance with all resource values
+ * initialized to {@code value}.
+ * @param value the value to use for all resources
+ * @return a new {@link Resource} instance
+ */
+ @Private
+ @Unstable
+ public static Resource newInstance(long value) {
+ return new LightWeightResource(value);
+ }
+
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
index a6e6432..77f77f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
@@ -18,9 +18,8 @@
package org.apache.hadoop.yarn.api.records.impl;
-import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -58,13 +57,29 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.*;
*
* @see Resource
*/
-@InterfaceAudience.Private
+@Private
@Unstable
public class LightWeightResource extends Resource {
private ResourceInformation memoryResInfo;
private ResourceInformation vcoresResInfo;
+ /**
+ * Create a new {@link LightWeightResource} instance with all resource values
+ * initialized to {@code value}.
+ * @param value the value to use for all resources
+ */
+ public LightWeightResource(long value) {
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
+ initResourceInformations(value, value, types.length);
+
+ for (int i = 2; i < types.length; i++) {
+ resources[i] = new ResourceInformation();
+ ResourceInformation.copy(types[i], resources[i]);
+ resources[i].setValue(value);
+ }
+ }
+
public LightWeightResource(long memory, int vcores) {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
@@ -91,7 +106,7 @@ public class LightWeightResource extends Resource {
}
}
- private void initResourceInformations(long memory, int vcores,
+ private void initResourceInformations(long memory, long vcores,
int numberOfKnownResourceTypes) {
this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(),
memory);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
index ecdd011..0c3b0dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
@@ -18,9 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
* A {@code ConfigurableResource} object represents an entity that is used to
@@ -33,29 +37,53 @@ public class ConfigurableResource {
private final Resource resource;
private final double[] percentages;
- public ConfigurableResource(double[] percentages) {
+ ConfigurableResource() {
+ this(getOneHundredPercentArray());
+ }
+
+ ConfigurableResource(double[] percentages) {
this.percentages = percentages.clone();
this.resource = null;
}
+ ConfigurableResource(long value) {
+ this(Resource.newInstance(value));
+ }
+
public ConfigurableResource(Resource resource) {
this.percentages = null;
this.resource = resource;
}
+ private static double[] getOneHundredPercentArray() {
+ double[] resourcePercentages =
+ new double[ResourceUtils.getNumberOfKnownResourceTypes()];
+ Arrays.fill(resourcePercentages, 1.0);
+
+ return resourcePercentages;
+ }
+
/**
* Get resource by multiplying the cluster resource and the percentage of
* each resource respectively. Return the absolute resource if either
* {@code percentages} or {@code clusterResource} is null.
*
* @param clusterResource the cluster resource
- * @return resource
+ * @return resource the resulting resource
*/
public Resource getResource(Resource clusterResource) {
if (percentages != null && clusterResource != null) {
long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
- return Resource.newInstance(memory, vcore);
+ Resource res = Resource.newInstance(memory, vcore);
+ ResourceInformation[] clusterInfo = clusterResource.getResources();
+
+ for (int i = 2; i < clusterInfo.length; i++) {
+ res.setResourceValue(i,
+ (long)(clusterInfo[i].getValue() * percentages[i]));
+ }
+
+ return res;
} else {
return resource;
}
@@ -69,4 +97,39 @@ public class ConfigurableResource {
public Resource getResource() {
return resource;
}
+
+ /**
+ * Set the value of the wrapped resource if this object isn't setup to use
+ * percentages. If this object is set to use percentages, this method has
+ * no effect.
+ *
+ * @param name the name of the resource
+ * @param value the value
+ */
+ void setValue(String name, long value) {
+ if (resource != null) {
+ resource.setResourceValue(name, value);
+ }
+ }
+
+ /**
+ * Set the percentage of the resource if this object is setup to use
+ * percentages. If this object is set to use percentages, this method has
+ * no effect.
+ *
+ * @param name the name of the resource
+ * @param value the percentage
+ */
+ void setPercentage(String name, double value) {
+ if (percentages != null) {
+ Integer index = ResourceUtils.getResourceTypeIndex().get(name);
+
+ if (index != null) {
+ percentages[index] = value;
+ } else {
+ throw new ResourceNotFoundException("The requested resource, \""
+ + name + "\", could not be found.");
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index b50e4bb..8c4932b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -213,6 +214,9 @@ public class FairSchedulerConfiguration extends Configuration {
CONF_PREFIX + "reservable-nodes";
public static final float RESERVABLE_NODES_DEFAULT = 0.05f;
+ private static final String INVALID_RESOURCE_DEFINITION_PREFIX =
+ "Error reading resource config--invalid resource definition: ";
+
public FairSchedulerConfiguration() {
super();
}
@@ -407,54 +411,167 @@ public class FairSchedulerConfiguration extends Configuration {
}
/**
- * Parses a resource config value of a form like "1024", "1024 mb",
- * or "1024 mb, 3 vcores". If no units are given, megabytes are assumed.
- *
- * @throws AllocationConfigurationException
+ * Parses a resource config value in one of three forms:
+ * <ol>
+ * <li>Percentage: "50%" or "40% memory, 60% cpu"</li>
+ * <li>New style resources: "vcores=10, memory-mb=1024"
+ * or "vcores=60%, memory-mb=40%"</li>
+ * <li>Old style resources: "1024 mb, 10 vcores"</li>
+ * </ol>
+ * In new style resources, any resource that is not specified will be
+ * set to {@link Long#MAX_VALUE} or 100%, as appropriate. Also, in the new
+ * style resources, units are not allowed. Units are assumed from the resource
+ * manager's settings for the resources when the value isn't a percentage.
+ *
+ * @param value the resource definition to parse
+ * @return a {@link ConfigurableResource} that represents the parsed value
+ * @throws AllocationConfigurationException if the raw value is not a valid
+ * resource definition
*/
- public static ConfigurableResource parseResourceConfigValue(String val)
+ public static ConfigurableResource parseResourceConfigValue(String value)
throws AllocationConfigurationException {
+ return parseResourceConfigValue(value, Long.MAX_VALUE);
+ }
+
+ /**
+ * Parses a resource config value in one of three forms:
+ * <ol>
+ * <li>Percentage: "50%" or "40% memory, 60% cpu"</li>
+ * <li>New style resources: "vcores=10, memory-mb=1024"
+ * or "vcores=60%, memory-mb=40%"</li>
+ * <li>Old style resources: "1024 mb, 10 vcores"</li>
+ * </ol>
+ * In new style resources, any resource that is not specified will be
+ * set to {@code missing} or 0%, as appropriate. Also, in the new style
+ * resources, units are not allowed. Units are assumed from the resource
+ * manager's settings for the resources when the value isn't a percentage.
+ *
+ * The {@code missing} parameter is only used in the case of new style
+ * resources without percentages. With new style resources with percentages,
+ * any missing resources will be assumed to be 100% because percentages are
+ * only used with maximum resource limits.
+ *
+ * @param value the resource definition to parse
+ * @param missing the value to use for any unspecified resources
+ * @return a {@link ConfigurableResource} that represents the parsed value
+ * @throws AllocationConfigurationException if the raw value is not a valid
+ * resource definition
+ */
+ public static ConfigurableResource parseResourceConfigValue(String value,
+ long missing) throws AllocationConfigurationException {
ConfigurableResource configurableResource;
+
+ if (value.trim().isEmpty()) {
+ throw new AllocationConfigurationException("Error reading resource "
+ + "config--the resource string is empty.");
+ }
+
try {
- val = StringUtils.toLowerCase(val);
- if (val.contains("%")) {
- configurableResource = new ConfigurableResource(
- getResourcePercentage(val));
+ if (value.contains("=")) {
+ configurableResource = parseNewStyleResource(value, missing);
+ } else if (value.contains("%")) {
+ configurableResource = parseOldStyleResourceAsPercentage(value);
} else {
- int memory = findResource(val, "mb");
- int vcores = findResource(val, "vcores");
- configurableResource = new ConfigurableResource(
- BuilderUtils.newResource(memory, vcores));
+ configurableResource = parseOldStyleResource(value);
}
- } catch (AllocationConfigurationException ex) {
- throw ex;
- } catch (Exception ex) {
+ } catch (RuntimeException ex) {
throw new AllocationConfigurationException(
"Error reading resource config", ex);
}
+
+ return configurableResource;
+ }
+
+ private static ConfigurableResource parseNewStyleResource(String value,
+ long missing) throws AllocationConfigurationException {
+
+ final ConfigurableResource configurableResource;
+ boolean asPercent = value.contains("%");
+ if (asPercent) {
+ configurableResource = new ConfigurableResource();
+ } else {
+ configurableResource = new ConfigurableResource(missing);
+ }
+
+ String[] resources = value.split(",");
+ for (String resource : resources) {
+ String[] parts = resource.split("=");
+
+ if (parts.length != 2) {
+ throw createConfigException(value,
+ "Every resource must be of the form: name=value.");
+ }
+
+ String resourceName = parts[0].trim();
+ String resourceValue = parts[1].trim();
+ try {
+ if (asPercent) {
+ configurableResource.setPercentage(resourceName,
+ findPercentage(resourceValue, ""));
+ } else {
+ configurableResource.setValue(resourceName,
+ Long.parseLong(resourceValue));
+ }
+ } catch (ResourceNotFoundException ex) {
+ throw createConfigException(value, "The "
+ + "resource name, \"" + resourceName + "\" was not "
+ + "recognized. Please check the value of "
+ + YarnConfiguration.RESOURCE_TYPES + " in the Resource "
+ + "Manager's configuration files.", ex);
+ } catch (NumberFormatException ex) {
+ // This only comes from Long.parseLong()
+ throw createConfigException(value, "The "
+ + "resource values must all be integers. \"" + resourceValue
+ + "\" is not an integer.", ex);
+ } catch (AllocationConfigurationException ex) {
+ // This only comes from findPercentage()
+ throw createConfigException(value, "The "
+ + "resource values must all be percentages. \""
+ + resourceValue + "\" is either not a number or does not "
+ + "include the '%' symbol.", ex);
+ }
+ }
return configurableResource;
}
+ private static ConfigurableResource parseOldStyleResourceAsPercentage(
+ String value) throws AllocationConfigurationException {
+ return new ConfigurableResource(
+ getResourcePercentage(StringUtils.toLowerCase(value)));
+ }
+
+ private static ConfigurableResource parseOldStyleResource(String value)
+ throws AllocationConfigurationException {
+ final String lCaseValue = StringUtils.toLowerCase(value);
+ int memory = findResource(lCaseValue, "mb");
+ int vcores = findResource(lCaseValue, "vcores");
+
+ return new ConfigurableResource(
+ BuilderUtils.newResource(memory, vcores));
+ }
+
private static double[] getResourcePercentage(
String val) throws AllocationConfigurationException {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
double[] resourcePercentage = new double[numberOfKnownResourceTypes];
String[] strings = val.split(",");
+
if (strings.length == 1) {
double percentage = findPercentage(strings[0], "");
for (int i = 0; i < numberOfKnownResourceTypes; i++) {
- resourcePercentage[i] = percentage/100;
+ resourcePercentage[i] = percentage;
}
} else {
- resourcePercentage[0] = findPercentage(val, "memory")/100;
- resourcePercentage[1] = findPercentage(val, "cpu")/100;
+ resourcePercentage[0] = findPercentage(val, "memory");
+ resourcePercentage[1] = findPercentage(val, "cpu");
}
+
return resourcePercentage;
}
private static double findPercentage(String val, String units)
- throws AllocationConfigurationException {
+ throws AllocationConfigurationException {
final Pattern pattern =
Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units);
Matcher matcher = pattern.matcher(val);
@@ -467,7 +584,22 @@ public class FairSchedulerConfiguration extends Configuration {
units);
}
}
- return Double.parseDouble(matcher.group(1));
+ return Double.parseDouble(matcher.group(1)) / 100.0;
+ }
+
+ private static AllocationConfigurationException createConfigException(
+ String value, String message) {
+ return createConfigException(value, message, null);
+ }
+
+ private static AllocationConfigurationException createConfigException(
+ String value, String message, Throwable t) {
+ String msg = INVALID_RESOURCE_DEFINITION_PREFIX + value + ". " + message;
+ if (t != null) {
+ return new AllocationConfigurationException(msg, t);
+ } else {
+ return new AllocationConfigurationException(msg);
+ }
}
public long getUpdateInterval() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
index d5a436e..441c34a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
@@ -134,7 +134,7 @@ public class AllocationFileQueueParser {
if (MIN_RESOURCES.equals(field.getTagName())) {
String text = getTrimmedTextData(field);
ConfigurableResource val =
- FairSchedulerConfiguration.parseResourceConfigValue(text);
+ FairSchedulerConfiguration.parseResourceConfigValue(text, 0L);
builder.minQueueResources(queueName, val.getResource());
} else if (MAX_RESOURCES.equals(field.getTagName())) {
String text = getTrimmedTextData(field);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 481645b..76a5af5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -102,60 +102,145 @@ public class TestFairSchedulerConfiguration {
@Test
public void testParseResourceConfigValue() throws Exception {
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("2 vcores, 1024 mb").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 mb, 2 vcores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("2vcores,1024mb").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024mb,2vcores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 mb, 2 vcores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("1024 Mb, 2 vCores").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource());
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource());
-
- Resource clusterResource = BuilderUtils.newResource(2048, 4);
- assertEquals(BuilderUtils.newResource(1024, 2),
+ Resource expected = BuilderUtils.newResource(5 * 1024, 2);
+ Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4);
+
+ assertEquals(expected,
+ parseResourceConfigValue("2 vcores, 5120 mb").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120 mb, 2 vcores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("2vcores,5120mb").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120mb,2vcores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120mb mb, 2 vcores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("5120 Mb, 2 vCores").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource());
+
+ assertEquals(expected,
parseResourceConfigValue("50% memory, 50% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50% Memory, 50% CpU").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
- parseResourceConfigValue("50%").getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 4),
+ assertEquals(BuilderUtils.newResource(5 * 1024, 4),
parseResourceConfigValue("50% memory, 100% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 4),
+ assertEquals(BuilderUtils.newResource(5 * 1024, 4),
parseResourceConfigValue(" 100% cpu, 50% memory").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 0),
+ assertEquals(BuilderUtils.newResource(5 * 1024, 0),
parseResourceConfigValue("50% memory, 0% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50 % memory, 50 % cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50%memory,50%cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue(" 50 % memory, 50 % cpu ").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(1024, 2),
+ assertEquals(expected,
parseResourceConfigValue("50.% memory, 50.% cpu").
getResource(clusterResource));
-
- clusterResource = BuilderUtils.newResource(1024 * 10, 4);
assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2),
parseResourceConfigValue("10.9% memory, 50.6% cpu").
getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("50%").getResource(clusterResource));
+
+ Configuration conf = new Configuration();
+
+ conf.set(YarnConfiguration.RESOURCE_TYPES, "test1");
+ ResourceUtils.resetResourceTypes(conf);
+
+ clusterResource = BuilderUtils.newResource(10 * 1024, 4);
+ expected = BuilderUtils.newResource(5 * 1024, 2);
+ expected.setResourceValue("test1", Long.MAX_VALUE);
+
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2, memory-mb=5120").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=5120, vcores=2").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2,memory-mb=5120").getResource());
+ assertEquals(expected, parseResourceConfigValue(" vcores = 2 , "
+ + "memory-mb = 5120 ").getResource());
+
+ expected.setResourceValue("test1", 0L);
+
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2, memory-mb=5120", 0L).getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=5120, vcores=2", 0L).getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2,memory-mb=5120", 0L).getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ",
+ 0L).getResource());
+
+ clusterResource.setResourceValue("test1", 8L);
+ expected.setResourceValue("test1", 4L);
+
+ assertEquals(expected,
+ parseResourceConfigValue("50%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2, memory-mb=5120, "
+ + "test1=4").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("test1=4, vcores=2, "
+ + "memory-mb=5120").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=5120, test1=4, "
+ + "vcores=2").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=2,memory-mb=5120,"
+ + "test1=4").getResource());
+ assertEquals(expected,
+ parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , "
+ + "test1 = 4 ").getResource());
+
+ expected = BuilderUtils.newResource(4 * 1024, 3);
+ expected.setResourceValue("test1", 8L);
+
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=75%, "
+ + "memory-mb=40%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=40%, "
+ + "vcores=75%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=75%,"
+ + "memory-mb=40%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue(" vcores = 75 % , "
+ + "memory-mb = 40 % ").getResource(clusterResource));
+
+ expected.setResourceValue("test1", 4L);
+
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=75%, memory-mb=40%, "
+ + "test1=50%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("test1=50%, vcores=75%, "
+ + "memory-mb=40%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("memory-mb=40%, test1=50%, "
+ + "vcores=75%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue("vcores=75%,memory-mb=40%,"
+ + "test1=50%").getResource(clusterResource));
+ assertEquals(expected,
+ parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , "
+ + "test1 = 50 % ").getResource(clusterResource));
}
@Test(expected = AllocationConfigurationException.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17262470/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index 269f5b4..b5bcbf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -86,11 +86,11 @@ The allocation file must be in XML format. The format contains five types of ele
* **Queue elements**: which represent queues. Queue elements can take an optional attribute 'type', which when set to 'parent' makes it a parent queue. This is useful when we want to create a parent queue without configuring any leaf queues. Each queue element may contain the following properties:
- * **minResources**: minimum resources the queue is entitled to, in the form "X mb, Y vcores". For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and minimum. Note that it is possible that a queue that is below its minimum may not immediately get up to its minimum when it submits an application, because already-running jobs may be using those resources.
+ * **minResources**: minimum resources the queue is entitled to, in the form of "X mb, Y vcores" or "vcores=X, memory-mb=Y". The latter form is required when specifying resources other than memory and CPU. For the single-resource fairness policy, the vcores value is ignored. If a queue's minimum share is not satisfied, it will be offered available resources before any other queue under the same parent. Under the single-resource fairness policy, a queue is considered unsatisfied if its memory usage is below its minimum memory share. Under dominant resource fairness, a queue is considered unsatisfied if its usage for its dominant resource with respect to the cluster capacity is below its minimum share for that resource. If multiple queues are unsatisfied in this situation, resources go to the queue with the smallest ratio between relevant resource usage and its minimum. Note that it is possible for a queue that is below its minimum to not immediately get up to its minimum when an a
pplication is submitted to the queue, because already-running jobs may be using those resources.
- * **maxResources**: maximum resources a queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). A queue will not be assigned a container that would put its aggregate usage over this limit.
+ * **maxResources**: maximum resources a queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. A queue will not be assigned a container that would put its aggregate usage over this limit.
- * **maxChildResources**: maximum resources an ad hoc child queue is allocated, expressed either in absolute values (X mb, Y vcores) or as a percentage of the cluster resources (X% memory, Y% cpu). An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
+ * **maxChildResources**: maximum resources an ad hoc child queue will allocated, expressed in the form of "X%", "X% cpu, Y% memory", "X mb, Y vcores", or "vcores=X, memory-mb=Y". The last form is required when specifying resources other than memory and CPU. In the last form, X and Y can either be a percentage or an integer resource value without units. In the latter case the units will be inferred from the default units configured for that resource. An ad hoc child queue will not be assigned a container that would put its aggregate usage over this limit.
* **maxRunningApps**: limit the number of apps from the queue to run at once
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org