You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2018/07/09 18:26:14 UTC
[22/50] [abbrv] hadoop git commit: Revert "Merge branch 'trunk' of
https://git-wip-us.apache.org/repos/asf/hadoop into trunk"
Revert "Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk"
This reverts commit c163d1797ade0f47d35b4a44381b8ef1dfec5b60, reversing
changes made to 0d9804dcef2eab5ebf84667d9ca49bb035d9a731.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39ad9890
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39ad9890
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39ad9890
Branch: refs/heads/HDFS-12090
Commit: 39ad98903a5f042573b97a2e5438bc57af7cc7a1
Parents: c163d17
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Jul 5 12:22:18 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Jul 5 12:22:18 2018 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 17 +-
.../hadoop/yarn/api/records/Resource.java | 13 -
.../api/records/impl/LightWeightResource.java | 23 +-
.../scheduler/fair/ConfigurableResource.java | 69 +----
.../fair/FairSchedulerConfiguration.java | 174 ++----------
.../allocation/AllocationFileQueueParser.java | 2 +-
.../resourcemanager/webapp/dao/AppInfo.java | 2 +-
.../webapp/dao/SchedulerInfo.java | 8 +-
.../fair/TestFairSchedulerConfiguration.java | 160 +++--------
.../webapp/TestRMWebServices.java | 31 +--
.../webapp/TestRMWebServicesApps.java | 14 +-
...estRMWebServicesAppsCustomResourceTypes.java | 242 -----------------
.../webapp/TestRMWebServicesCapacitySched.java | 30 +-
.../TestRMWebServicesConfigurationMutation.java | 5 -
.../webapp/TestRMWebServicesFairScheduler.java | 95 ++++---
.../TestRMWebServicesSchedulerActivities.java | 2 +-
...ustomResourceTypesConfigurationProvider.java | 138 ----------
.../FairSchedulerJsonVerifications.java | 139 ----------
.../FairSchedulerXmlVerifications.java | 153 -----------
...ervicesFairSchedulerCustomResourceTypes.java | 271 -------------------
.../webapp/helper/AppInfoJsonVerifications.java | 123 ---------
.../webapp/helper/AppInfoXmlVerifications.java | 132 ---------
.../webapp/helper/BufferedClientResponse.java | 57 ----
.../helper/JsonCustomResourceTypeTestcase.java | 77 ------
.../ResourceRequestsJsonVerifications.java | 252 -----------------
.../ResourceRequestsXmlVerifications.java | 215 ---------------
.../helper/XmlCustomResourceTypeTestCase.java | 112 --------
.../src/site/markdown/FairScheduler.md | 6 +-
28 files changed, 157 insertions(+), 2405 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 5cc81e5..5841361 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -67,6 +67,11 @@
</Match>
<Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
+ <Method name="getLocalityStatistics" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
<Method name="incNumAllocatedContainers"/>
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
@@ -113,18 +118,6 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
- <!-- Ignore exposed internal representations -->
- <Match>
- <Class name="org.apache.hadoop.yarn.api.records.Resource" />
- <Method name="getResources" />
- <Bug pattern="EI_EXPOSE_REP" />
- </Match>
- <Match>
- <Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics" />
- <Method name="getLocalityStatistics" />
- <Bug pattern="EI_EXPOSE_REP" />
- </Match>
-
<!-- Object cast is based on the event type -->
<Match>
<Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" />
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 173d4c9..71a6b54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.impl.LightWeightResource;
@@ -76,18 +75,6 @@ public abstract class Resource implements Comparable<Resource> {
@Private
public static final int VCORES_INDEX = 1;
- /**
- * Return a new {@link Resource} instance with all resource values
- * initialized to {@code value}.
- * @param value the value to use for all resources
- * @return a new {@link Resource} instance
- */
- @Private
- @Unstable
- public static Resource newInstance(long value) {
- return new LightWeightResource(value);
- }
-
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
index 77f77f3..a6e6432 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/LightWeightResource.java
@@ -18,8 +18,9 @@
package org.apache.hadoop.yarn.api.records.impl;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -57,29 +58,13 @@ import static org.apache.hadoop.yarn.api.records.ResourceInformation.*;
*
* @see Resource
*/
-@Private
+@InterfaceAudience.Private
@Unstable
public class LightWeightResource extends Resource {
private ResourceInformation memoryResInfo;
private ResourceInformation vcoresResInfo;
- /**
- * Create a new {@link LightWeightResource} instance with all resource values
- * initialized to {@code value}.
- * @param value the value to use for all resources
- */
- public LightWeightResource(long value) {
- ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
- initResourceInformations(value, value, types.length);
-
- for (int i = 2; i < types.length; i++) {
- resources[i] = new ResourceInformation();
- ResourceInformation.copy(types[i], resources[i]);
- resources[i].setValue(value);
- }
- }
-
public LightWeightResource(long memory, int vcores) {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
@@ -106,7 +91,7 @@ public class LightWeightResource extends Resource {
}
}
- private void initResourceInformations(long memory, long vcores,
+ private void initResourceInformations(long memory, int vcores,
int numberOfKnownResourceTypes) {
this.memoryResInfo = newDefaultInformation(MEMORY_URI, MEMORY_MB.getUnits(),
memory);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
index 0c3b0dd..ecdd011 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java
@@ -18,13 +18,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
* A {@code ConfigurableResource} object represents an entity that is used to
@@ -37,53 +33,29 @@ public class ConfigurableResource {
private final Resource resource;
private final double[] percentages;
- ConfigurableResource() {
- this(getOneHundredPercentArray());
- }
-
- ConfigurableResource(double[] percentages) {
+ public ConfigurableResource(double[] percentages) {
this.percentages = percentages.clone();
this.resource = null;
}
- ConfigurableResource(long value) {
- this(Resource.newInstance(value));
- }
-
public ConfigurableResource(Resource resource) {
this.percentages = null;
this.resource = resource;
}
- private static double[] getOneHundredPercentArray() {
- double[] resourcePercentages =
- new double[ResourceUtils.getNumberOfKnownResourceTypes()];
- Arrays.fill(resourcePercentages, 1.0);
-
- return resourcePercentages;
- }
-
/**
* Get resource by multiplying the cluster resource and the percentage of
* each resource respectively. Return the absolute resource if either
* {@code percentages} or {@code clusterResource} is null.
*
* @param clusterResource the cluster resource
- * @return resource the resulting resource
+ * @return resource
*/
public Resource getResource(Resource clusterResource) {
if (percentages != null && clusterResource != null) {
long memory = (long) (clusterResource.getMemorySize() * percentages[0]);
int vcore = (int) (clusterResource.getVirtualCores() * percentages[1]);
- Resource res = Resource.newInstance(memory, vcore);
- ResourceInformation[] clusterInfo = clusterResource.getResources();
-
- for (int i = 2; i < clusterInfo.length; i++) {
- res.setResourceValue(i,
- (long)(clusterInfo[i].getValue() * percentages[i]));
- }
-
- return res;
+ return Resource.newInstance(memory, vcore);
} else {
return resource;
}
@@ -97,39 +69,4 @@ public class ConfigurableResource {
public Resource getResource() {
return resource;
}
-
- /**
- * Set the value of the wrapped resource if this object isn't setup to use
- * percentages. If this object is set to use percentages, this method has
- * no effect.
- *
- * @param name the name of the resource
- * @param value the value
- */
- void setValue(String name, long value) {
- if (resource != null) {
- resource.setResourceValue(name, value);
- }
- }
-
- /**
- * Set the percentage of the resource if this object is setup to use
- * percentages. If this object is set to use percentages, this method has
- * no effect.
- *
- * @param name the name of the resource
- * @param value the percentage
- */
- void setPercentage(String name, double value) {
- if (percentages != null) {
- Integer index = ResourceUtils.getResourceTypeIndex().get(name);
-
- if (index != null) {
- percentages[index] = value;
- } else {
- throw new ResourceNotFoundException("The requested resource, \""
- + name + "\", could not be found.");
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 8c4932b..b50e4bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
@@ -214,9 +213,6 @@ public class FairSchedulerConfiguration extends Configuration {
CONF_PREFIX + "reservable-nodes";
public static final float RESERVABLE_NODES_DEFAULT = 0.05f;
- private static final String INVALID_RESOURCE_DEFINITION_PREFIX =
- "Error reading resource config--invalid resource definition: ";
-
public FairSchedulerConfiguration() {
super();
}
@@ -411,167 +407,54 @@ public class FairSchedulerConfiguration extends Configuration {
}
/**
- * Parses a resource config value in one of three forms:
- * <ol>
- * <li>Percentage: "50%" or "40% memory, 60% cpu"</li>
- * <li>New style resources: "vcores=10, memory-mb=1024"
- * or "vcores=60%, memory-mb=40%"</li>
- * <li>Old style resources: "1024 mb, 10 vcores"</li>
- * </ol>
- * In new style resources, any resource that is not specified will be
- * set to {@link Long#MAX_VALUE} or 100%, as appropriate. Also, in the new
- * style resources, units are not allowed. Units are assumed from the resource
- * manager's settings for the resources when the value isn't a percentage.
- *
- * @param value the resource definition to parse
- * @return a {@link ConfigurableResource} that represents the parsed value
- * @throws AllocationConfigurationException if the raw value is not a valid
- * resource definition
+ * Parses a resource config value of a form like "1024", "1024 mb",
+ * or "1024 mb, 3 vcores". If no units are given, megabytes are assumed.
+ *
+ * @throws AllocationConfigurationException
*/
- public static ConfigurableResource parseResourceConfigValue(String value)
+ public static ConfigurableResource parseResourceConfigValue(String val)
throws AllocationConfigurationException {
- return parseResourceConfigValue(value, Long.MAX_VALUE);
- }
-
- /**
- * Parses a resource config value in one of three forms:
- * <ol>
- * <li>Percentage: "50%" or "40% memory, 60% cpu"</li>
- * <li>New style resources: "vcores=10, memory-mb=1024"
- * or "vcores=60%, memory-mb=40%"</li>
- * <li>Old style resources: "1024 mb, 10 vcores"</li>
- * </ol>
- * In new style resources, any resource that is not specified will be
- * set to {@code missing} or 0%, as appropriate. Also, in the new style
- * resources, units are not allowed. Units are assumed from the resource
- * manager's settings for the resources when the value isn't a percentage.
- *
- * The {@code missing} parameter is only used in the case of new style
- * resources without percentages. With new style resources with percentages,
- * any missing resources will be assumed to be 100% because percentages are
- * only used with maximum resource limits.
- *
- * @param value the resource definition to parse
- * @param missing the value to use for any unspecified resources
- * @return a {@link ConfigurableResource} that represents the parsed value
- * @throws AllocationConfigurationException if the raw value is not a valid
- * resource definition
- */
- public static ConfigurableResource parseResourceConfigValue(String value,
- long missing) throws AllocationConfigurationException {
ConfigurableResource configurableResource;
-
- if (value.trim().isEmpty()) {
- throw new AllocationConfigurationException("Error reading resource "
- + "config--the resource string is empty.");
- }
-
try {
- if (value.contains("=")) {
- configurableResource = parseNewStyleResource(value, missing);
- } else if (value.contains("%")) {
- configurableResource = parseOldStyleResourceAsPercentage(value);
+ val = StringUtils.toLowerCase(val);
+ if (val.contains("%")) {
+ configurableResource = new ConfigurableResource(
+ getResourcePercentage(val));
} else {
- configurableResource = parseOldStyleResource(value);
+ int memory = findResource(val, "mb");
+ int vcores = findResource(val, "vcores");
+ configurableResource = new ConfigurableResource(
+ BuilderUtils.newResource(memory, vcores));
}
- } catch (RuntimeException ex) {
+ } catch (AllocationConfigurationException ex) {
+ throw ex;
+ } catch (Exception ex) {
throw new AllocationConfigurationException(
"Error reading resource config", ex);
}
-
- return configurableResource;
- }
-
- private static ConfigurableResource parseNewStyleResource(String value,
- long missing) throws AllocationConfigurationException {
-
- final ConfigurableResource configurableResource;
- boolean asPercent = value.contains("%");
- if (asPercent) {
- configurableResource = new ConfigurableResource();
- } else {
- configurableResource = new ConfigurableResource(missing);
- }
-
- String[] resources = value.split(",");
- for (String resource : resources) {
- String[] parts = resource.split("=");
-
- if (parts.length != 2) {
- throw createConfigException(value,
- "Every resource must be of the form: name=value.");
- }
-
- String resourceName = parts[0].trim();
- String resourceValue = parts[1].trim();
- try {
- if (asPercent) {
- configurableResource.setPercentage(resourceName,
- findPercentage(resourceValue, ""));
- } else {
- configurableResource.setValue(resourceName,
- Long.parseLong(resourceValue));
- }
- } catch (ResourceNotFoundException ex) {
- throw createConfigException(value, "The "
- + "resource name, \"" + resourceName + "\" was not "
- + "recognized. Please check the value of "
- + YarnConfiguration.RESOURCE_TYPES + " in the Resource "
- + "Manager's configuration files.", ex);
- } catch (NumberFormatException ex) {
- // This only comes from Long.parseLong()
- throw createConfigException(value, "The "
- + "resource values must all be integers. \"" + resourceValue
- + "\" is not an integer.", ex);
- } catch (AllocationConfigurationException ex) {
- // This only comes from findPercentage()
- throw createConfigException(value, "The "
- + "resource values must all be percentages. \""
- + resourceValue + "\" is either not a number or does not "
- + "include the '%' symbol.", ex);
- }
- }
return configurableResource;
}
- private static ConfigurableResource parseOldStyleResourceAsPercentage(
- String value) throws AllocationConfigurationException {
- return new ConfigurableResource(
- getResourcePercentage(StringUtils.toLowerCase(value)));
- }
-
- private static ConfigurableResource parseOldStyleResource(String value)
- throws AllocationConfigurationException {
- final String lCaseValue = StringUtils.toLowerCase(value);
- int memory = findResource(lCaseValue, "mb");
- int vcores = findResource(lCaseValue, "vcores");
-
- return new ConfigurableResource(
- BuilderUtils.newResource(memory, vcores));
- }
-
private static double[] getResourcePercentage(
String val) throws AllocationConfigurationException {
int numberOfKnownResourceTypes = ResourceUtils
.getNumberOfKnownResourceTypes();
double[] resourcePercentage = new double[numberOfKnownResourceTypes];
String[] strings = val.split(",");
-
if (strings.length == 1) {
double percentage = findPercentage(strings[0], "");
for (int i = 0; i < numberOfKnownResourceTypes; i++) {
- resourcePercentage[i] = percentage;
+ resourcePercentage[i] = percentage/100;
}
} else {
- resourcePercentage[0] = findPercentage(val, "memory");
- resourcePercentage[1] = findPercentage(val, "cpu");
+ resourcePercentage[0] = findPercentage(val, "memory")/100;
+ resourcePercentage[1] = findPercentage(val, "cpu")/100;
}
-
return resourcePercentage;
}
private static double findPercentage(String val, String units)
- throws AllocationConfigurationException {
+ throws AllocationConfigurationException {
final Pattern pattern =
Pattern.compile("((\\d+)(\\.\\d*)?)\\s*%\\s*" + units);
Matcher matcher = pattern.matcher(val);
@@ -584,22 +467,7 @@ public class FairSchedulerConfiguration extends Configuration {
units);
}
}
- return Double.parseDouble(matcher.group(1)) / 100.0;
- }
-
- private static AllocationConfigurationException createConfigException(
- String value, String message) {
- return createConfigException(value, message, null);
- }
-
- private static AllocationConfigurationException createConfigException(
- String value, String message, Throwable t) {
- String msg = INVALID_RESOURCE_DEFINITION_PREFIX + value + ". " + message;
- if (t != null) {
- return new AllocationConfigurationException(msg, t);
- } else {
- return new AllocationConfigurationException(msg);
- }
+ return Double.parseDouble(matcher.group(1));
}
public long getUpdateInterval() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
index 441c34a..d5a436e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
@@ -134,7 +134,7 @@ public class AllocationFileQueueParser {
if (MIN_RESOURCES.equals(field.getTagName())) {
String text = getTrimmedTextData(field);
ConfigurableResource val =
- FairSchedulerConfiguration.parseResourceConfigValue(text, 0L);
+ FairSchedulerConfiguration.parseResourceConfigValue(text);
builder.minQueueResources(queueName, val.getResource());
} else if (MAX_RESOURCES.equals(field.getTagName())) {
String text = getTrimmedTextData(field);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 9d82bc7..d47f13d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -479,7 +479,7 @@ public class AppInfo {
public int getNumNonAMContainersPreempted() {
return numNonAMContainerPreempted;
}
-
+
public int getNumAMContainersPreempted() {
return numAMContainerPreempted;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 163f707..81491b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -41,9 +41,8 @@ public class SchedulerInfo {
protected EnumSet<SchedulerResourceTypes> schedulingResourceTypes;
protected int maximumClusterPriority;
- // JAXB needs this
public SchedulerInfo() {
- }
+ } // JAXB needs this
public SchedulerInfo(final ResourceManager rm) {
ResourceScheduler rs = rm.getResourceScheduler();
@@ -75,10 +74,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
- if (minAllocResource != null) {
- return Arrays.toString(minAllocResource.getResource().getResources());
- }
- return null;
+ return Arrays.toString(minAllocResource.getResource().getResources());
}
public int getMaxClusterLevelAppPriority() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 70f83ab..481645b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -48,9 +48,6 @@ import org.apache.log4j.spi.LoggingEvent;
import org.junit.Assert;
import org.junit.Test;
-/**
- * Tests fair scheduler configuration.
- */
public class TestFairSchedulerConfiguration {
private static final String A_CUSTOM_RESOURCE = "a-custom-resource";
@@ -105,152 +102,67 @@ public class TestFairSchedulerConfiguration {
@Test
public void testParseResourceConfigValue() throws Exception {
- Resource expected = BuilderUtils.newResource(5 * 1024, 2);
- Resource clusterResource = BuilderUtils.newResource(10 * 1024, 4);
-
- assertEquals(expected,
- parseResourceConfigValue("2 vcores, 5120 mb").getResource());
- assertEquals(expected,
- parseResourceConfigValue("5120 mb, 2 vcores").getResource());
- assertEquals(expected,
- parseResourceConfigValue("2vcores,5120mb").getResource());
- assertEquals(expected,
- parseResourceConfigValue("5120mb,2vcores").getResource());
- assertEquals(expected,
- parseResourceConfigValue("5120mb mb, 2 vcores").getResource());
- assertEquals(expected,
- parseResourceConfigValue("5120 Mb, 2 vCores").getResource());
- assertEquals(expected,
- parseResourceConfigValue(" 5120 mb, 2 vcores ").getResource());
- assertEquals(expected,
- parseResourceConfigValue(" 5120.3 mb, 2.35 vcores ").getResource());
- assertEquals(expected,
- parseResourceConfigValue(" 5120. mb, 2. vcores ").getResource());
-
- assertEquals(expected,
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("2 vcores, 1024 mb").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024 mb, 2 vcores").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("2vcores,1024mb").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024mb,2vcores").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024 mb, 2 vcores").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("1024 Mb, 2 vCores").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue(" 1024 mb, 2 vcores ").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue(" 1024.3 mb, 2.35 vcores ").getResource());
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue(" 1024. mb, 2. vcores ").getResource());
+
+ Resource clusterResource = BuilderUtils.newResource(2048, 4);
+ assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50% memory, 50% cpu").
getResource(clusterResource));
- assertEquals(expected,
+ assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50% Memory, 50% CpU").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(5 * 1024, 4),
+ assertEquals(BuilderUtils.newResource(1024, 2),
+ parseResourceConfigValue("50%").getResource(clusterResource));
+ assertEquals(BuilderUtils.newResource(1024, 4),
parseResourceConfigValue("50% memory, 100% cpu").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(5 * 1024, 4),
+ assertEquals(BuilderUtils.newResource(1024, 4),
parseResourceConfigValue(" 100% cpu, 50% memory").
getResource(clusterResource));
- assertEquals(BuilderUtils.newResource(5 * 1024, 0),
+ assertEquals(BuilderUtils.newResource(1024, 0),
parseResourceConfigValue("50% memory, 0% cpu").
getResource(clusterResource));
- assertEquals(expected,
+ assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50 % memory, 50 % cpu").
getResource(clusterResource));
- assertEquals(expected,
+ assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50%memory,50%cpu").
getResource(clusterResource));
- assertEquals(expected,
+ assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue(" 50 % memory, 50 % cpu ").
getResource(clusterResource));
- assertEquals(expected,
+ assertEquals(BuilderUtils.newResource(1024, 2),
parseResourceConfigValue("50.% memory, 50.% cpu").
getResource(clusterResource));
+
+ clusterResource = BuilderUtils.newResource(1024 * 10, 4);
assertEquals(BuilderUtils.newResource((int)(1024 * 10 * 0.109), 2),
parseResourceConfigValue("10.9% memory, 50.6% cpu").
getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("50%").getResource(clusterResource));
-
- Configuration conf = new Configuration();
-
- conf.set(YarnConfiguration.RESOURCE_TYPES, "test1");
- ResourceUtils.resetResourceTypes(conf);
-
- clusterResource = BuilderUtils.newResource(10 * 1024, 4);
- expected = BuilderUtils.newResource(5 * 1024, 2);
- expected.setResourceValue("test1", Long.MAX_VALUE);
-
- assertEquals(expected,
- parseResourceConfigValue("vcores=2, memory-mb=5120").getResource());
- assertEquals(expected,
- parseResourceConfigValue("memory-mb=5120, vcores=2").getResource());
- assertEquals(expected,
- parseResourceConfigValue("vcores=2,memory-mb=5120").getResource());
- assertEquals(expected, parseResourceConfigValue(" vcores = 2 , "
- + "memory-mb = 5120 ").getResource());
-
- expected.setResourceValue("test1", 0L);
-
- assertEquals(expected,
- parseResourceConfigValue("vcores=2, memory-mb=5120", 0L).getResource());
- assertEquals(expected,
- parseResourceConfigValue("memory-mb=5120, vcores=2", 0L).getResource());
- assertEquals(expected,
- parseResourceConfigValue("vcores=2,memory-mb=5120", 0L).getResource());
- assertEquals(expected,
- parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 ",
- 0L).getResource());
-
- clusterResource.setResourceValue("test1", 8L);
- expected.setResourceValue("test1", 4L);
-
- assertEquals(expected,
- parseResourceConfigValue("50%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("vcores=2, memory-mb=5120, "
- + "test1=4").getResource());
- assertEquals(expected,
- parseResourceConfigValue("test1=4, vcores=2, "
- + "memory-mb=5120").getResource());
- assertEquals(expected,
- parseResourceConfigValue("memory-mb=5120, test1=4, "
- + "vcores=2").getResource());
- assertEquals(expected,
- parseResourceConfigValue("vcores=2,memory-mb=5120,"
- + "test1=4").getResource());
- assertEquals(expected,
- parseResourceConfigValue(" vcores = 2 , memory-mb = 5120 , "
- + "test1 = 4 ").getResource());
-
- expected = BuilderUtils.newResource(4 * 1024, 3);
- expected.setResourceValue("test1", 8L);
-
- assertEquals(expected,
- parseResourceConfigValue("vcores=75%, "
- + "memory-mb=40%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("memory-mb=40%, "
- + "vcores=75%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("vcores=75%,"
- + "memory-mb=40%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue(" vcores = 75 % , "
- + "memory-mb = 40 % ").getResource(clusterResource));
-
- expected.setResourceValue("test1", 4L);
-
- assertEquals(expected,
- parseResourceConfigValue("vcores=75%, memory-mb=40%, "
- + "test1=50%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("test1=50%, vcores=75%, "
- + "memory-mb=40%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("memory-mb=40%, test1=50%, "
- + "vcores=75%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue("vcores=75%,memory-mb=40%,"
- + "test1=50%").getResource(clusterResource));
- assertEquals(expected,
- parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , "
- + "test1 = 50 % ").getResource(clusterResource));
}
-
+
@Test(expected = AllocationConfigurationException.class)
public void testNoUnits() throws Exception {
parseResourceConfigValue("1024");
}
-
+
@Test(expected = AllocationConfigurationException.class)
public void testOnlyMemory() throws Exception {
parseResourceConfigValue("1024mb");
@@ -260,7 +172,7 @@ public class TestFairSchedulerConfiguration {
public void testOnlyCPU() throws Exception {
parseResourceConfigValue("1024vcores");
}
-
+
@Test(expected = AllocationConfigurationException.class)
public void testGibberish() throws Exception {
parseResourceConfigValue("1o24vc0res");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 3902889..0702d65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -53,7 +53,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.*;
+import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
+import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -72,12 +76,11 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
+import org.eclipse.jetty.server.Response;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -93,8 +96,6 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServices extends JerseyTestBase {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestRMWebServices.class);
private static MockRM rm;
@@ -471,19 +472,19 @@ public class TestRMWebServices extends JerseyTestBase {
QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
- long totalMBExpect =
+ long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
- long totalVirtualCoresExpect =
+ long totalVirtualCoresExpect =
metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
- assertEquals("appsSubmitted doesn't match",
+ assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
- assertEquals("appsCompleted doesn't match",
+ assertEquals("appsCompleted doesn't match",
metrics.getAppsCompleted(), completedApps);
assertEquals("reservedMB doesn't match",
metrics.getReservedMB(), reservedMB);
- assertEquals("availableMB doesn't match",
+ assertEquals("availableMB doesn't match",
metrics.getAvailableMB(), availableMB);
- assertEquals("allocatedMB doesn't match",
+ assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
assertEquals("reservedVirtualCores doesn't match",
metrics.getReservedVirtualCores(), reservedVirtualCores);
@@ -596,13 +597,11 @@ public class TestRMWebServices extends JerseyTestBase {
public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException,
Exception {
- assertEquals("incorrect number of elements in: " + json, 1, json.length());
+ assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
- assertEquals("incorrect number of elements in: " + info, 1, info.length());
+ assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
-
- LOG.debug("schedulerInfo: {}", info);
- assertEquals("incorrect number of elements in: " + info, 11, info.length());
+ assertEquals("incorrect number of elements", 11, info.length());
verifyClusterSchedulerFifoGeneric(info.getString("type"),
info.getString("qstate"), (float) info.getDouble("capacity"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 15f94e1..6c6f400 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -79,7 +79,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServicesApps extends JerseyTestBase {
private static MockRM rm;
-
+
private static final int CONTAINER_MB = 1024;
private static class WebServletModule extends ServletModule {
@@ -324,7 +324,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
- assertTrue("both app states of ACCEPTED and KILLED are not present",
+ assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
@@ -375,12 +375,12 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
- assertTrue("both app states of ACCEPTED and KILLED are not present",
+ assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
array.getJSONObject(1).getString("state").equals("ACCEPTED")));
-
+
rm.stop();
}
@@ -511,8 +511,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").queryParam("finalStatus",
- FinalApplicationStatus.UNDEFINED.toString())
+ .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
@@ -1805,8 +1804,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
int numAttempt = 1;
while (true) {
// fail the AM by sending CONTAINER_FINISHED event without registering.
- amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
- ContainerState.COMPLETE);
+ amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
if (numAttempt == maxAppAttempts) {
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
deleted file mode 100644
index 83e0056..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
-import org.apache.hadoop.yarn.util.resource.ResourceUtils;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
-import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.codehaus.jettison.json.JSONArray;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Before;
-import org.junit.Test;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-
-import javax.ws.rs.core.MediaType;
-import java.util.ArrayList;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * This test verifies that custom resource types are correctly serialized to XML
- * and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
- */
-public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
-
- private static MockRM rm;
- private static final int CONTAINER_MB = 1024;
-
- private static class WebServletModule extends ServletModule {
- @Override
- protected void configureServlets() {
- bind(JAXBContextResolver.class);
- bind(RMWebServices.class);
- bind(GenericExceptionHandler.class);
- Configuration conf = new Configuration();
- conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
- YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
- ResourceScheduler.class);
- initResourceTypes(conf);
- rm = new MockRM(conf);
- bind(ResourceManager.class).toInstance(rm);
- serve("/*").with(GuiceContainer.class);
- }
-
- private void initResourceTypes(Configuration conf) {
- conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
- CustomResourceTypesConfigurationProvider.class.getName());
- ResourceUtils.resetResourceTypes(conf);
- }
- }
-
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- createInjectorForWebServletModule();
- }
-
- private void createInjectorForWebServletModule() {
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
- }
-
- public TestRMWebServicesAppsCustomResourceTypes() {
- super(new WebAppDescriptor.Builder(
- "org.apache.hadoop.yarn.server.resourcemanager.webapp")
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class)
- .contextPath("jersey-guice-filter").servletPath("/").build());
- }
-
- @Test
- public void testRunningAppXml() throws Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
- am1.allocate("*", 2048, 1, new ArrayList<>());
- amNodeManager.nodeHeartbeat(true);
-
- WebResource r = resource();
- WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
- ClientResponse response =
- path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
-
- XmlCustomResourceTypeTestCase testCase =
- new XmlCustomResourceTypeTestCase(path,
- new BufferedClientResponse(response));
- testCase.verify(document -> {
- NodeList apps = document.getElementsByTagName("apps");
- assertEquals("incorrect number of apps elements", 1, apps.getLength());
-
- NodeList appArray = ((Element)(apps.item(0)))
- .getElementsByTagName("app");
- assertEquals("incorrect number of app elements", 1, appArray.getLength());
-
- verifyAppsXML(appArray, app1);
- });
-
- rm.stop();
- }
-
- @Test
- public void testRunningAppJson() throws Exception {
- rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
- RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
- am1.allocate("*", 2048, 1, new ArrayList<>());
- amNodeManager.nodeHeartbeat(true);
-
- WebResource r = resource();
- WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
- ClientResponse response =
- path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
-
- JsonCustomResourceTypeTestcase testCase =
- new JsonCustomResourceTypeTestcase(path,
- new BufferedClientResponse(response));
- testCase.verify(json -> {
- try {
- assertEquals("incorrect number of apps elements", 1, json.length());
- JSONObject apps = json.getJSONObject("apps");
- assertEquals("incorrect number of app elements", 1, apps.length());
- JSONArray array = apps.getJSONArray("app");
- assertEquals("incorrect count of app", 1, array.length());
-
- verifyAppInfoJson(array.getJSONObject(0), app1);
- } catch (JSONException e) {
- throw new RuntimeException(e);
- }
- });
-
- rm.stop();
- }
-
- private void verifyAppsXML(NodeList appArray, RMApp app) {
- for (int i = 0; i < appArray.getLength(); i++) {
- Element element = (Element) appArray.item(i);
- AppInfoXmlVerifications.verify(element, app);
-
- NodeList resourceRequests =
- element.getElementsByTagName("resourceRequests");
- assertEquals(1, resourceRequests.getLength());
- Node resourceRequest = resourceRequests.item(0);
- ResourceRequest rr =
- ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
- .getApplicationAttempt(
- app.getCurrentAppAttempt().getAppAttemptId())
- .getAppSchedulingInfo().getAllResourceRequests().get(0);
- ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
- (Element) resourceRequest, rr,
- CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
- }
- }
-
- private void verifyAppInfoJson(JSONObject info, RMApp app) throws
- JSONException {
- int expectedNumberOfElements = getExpectedNumberOfElements(app);
-
- assertEquals("incorrect number of elements", expectedNumberOfElements,
- info.length());
-
- AppInfoJsonVerifications.verify(info, app);
-
- JSONArray resourceRequests = info.getJSONArray("resourceRequests");
- JSONObject requestInfo = resourceRequests.getJSONObject(0);
- ResourceRequest rr =
- ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
- .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
- .getAppSchedulingInfo().getAllResourceRequests().get(0);
-
- ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
- requestInfo, rr,
- CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
- }
-
- private int getExpectedNumberOfElements(RMApp app) {
- int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
- if (app.getApplicationSubmissionContext()
- .getNodeLabelExpression() != null) {
- expectedNumberOfElements++;
- }
-
- if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
- expectedNumberOfElements++;
- }
-
- if (AppInfo
- .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
- expectedNumberOfElements++;
- }
- return expectedNumberOfElements;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 46d0a66..e37f76f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -146,7 +146,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
config.setUserLimitFactor(B2, 100.0f);
config.setCapacity(B3, 0.5f);
config.setUserLimitFactor(B3, 100.0f);
-
+
config.setQueues(A1, new String[] {"a1a", "a1b"});
final String A1A = A1 + ".a1a";
config.setCapacity(A1A, 85);
@@ -254,7 +254,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
}
}
- public void verifySubQueueXML(Element qElem, String q,
+ public void verifySubQueueXML(Element qElem, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws Exception {
NodeList children = qElem.getChildNodes();
@@ -317,34 +317,30 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
- assertEquals("incorrect number of elements in: " + json, 1, json.length());
+ assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
- assertEquals("incorrect number of elements in: " + info, 1, info.length());
+ assertEquals("incorrect number of elements", 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements in: " + info, 8, info.length());
+ assertEquals("incorrect number of elements", 8, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), info.getString("queueName"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
- assertEquals("incorrect number of elements in: " + health, 3,
- health.length());
+ assertEquals("incorrect number of elements", 3, health.length());
JSONArray operationsInfo = health.getJSONArray("operationsInfo");
- assertEquals("incorrect number of elements in: " + health, 4,
- operationsInfo.length());
+ assertEquals("incorrect number of elements", 4, operationsInfo.length());
JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
- assertEquals("incorrect number of elements in: " + health, 3,
- lastRunDetails.length());
+ assertEquals("incorrect number of elements", 3, lastRunDetails.length());
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
- assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
+ assertEquals("incorrect number of elements", 2, arr.length());
// test subqueues
for (int i = 0; i < arr.length(); i++) {
JSONObject obj = arr.getJSONObject(i);
- String q = CapacitySchedulerConfiguration.ROOT + "." +
- obj.getString("queueName");
+ String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName");
verifySubQueue(obj, q, 100, 100);
}
}
@@ -359,7 +355,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
assertTrue("queueName doesn't match", "root".matches(queueName));
}
- private void verifySubQueue(JSONObject info, String q,
+ private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 20;
@@ -468,7 +464,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f);
}
- //Return a child Node of node with the tagname or null if none exists
+ //Return a child Node of node with the tagname or null if none exists
private Node getChildNodeByName(Node node, String tagname) {
NodeList nodeList = node.getChildNodes();
for (int i=0; i < nodeList.getLength(); ++i) {
@@ -518,7 +514,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
for (int j=0; j<users.getLength(); ++j) {
Node user = users.item(j);
String username = getChildNodeByName(user, "username")
- .getTextContent();
+ .getTextContent();
assertTrue(username.equals("user1") || username.equals("user2"));
//Should be a parsable integer
Integer.parseInt(getChildNodeByName(getChildNodeByName(user,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 99b5648..3d28f12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -42,8 +42,6 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
@@ -61,8 +59,6 @@ import static org.junit.Assert.assertNull;
* Test scheduler configuration mutation via REST API.
*/
public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
- private static final Logger LOG = LoggerFactory
- .getLogger(TestRMWebServicesConfigurationMutation.class);
private static final File CONF_FILE = new File(new File("target",
"test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@@ -400,7 +396,6 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
- LOG.debug("Response headers: " + response.getHeaders());
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
assertEquals(0.2f, newCSConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
index 58c72ee..e77785b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,14 +16,13 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import javax.ws.rs.core.MediaType;
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -31,9 +30,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
-
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -42,18 +38,18 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
-import javax.ws.rs.core.MediaType;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
-/**
- * Tests RM Webservices fair scheduler resources.
- */
public class TestRMWebServicesFairScheduler extends JerseyTestBase {
private static MockRM rm;
private static YarnConfiguration conf;
-
+
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
@@ -62,7 +58,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
bind(GenericExceptionHandler.class);
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
- ResourceScheduler.class);
+ ResourceScheduler.class);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
@@ -70,32 +66,32 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
static {
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- GuiceServletConfig
- .setInjector(Guice.createInjector(new WebServletModule()));
+ GuiceServletConfig.setInjector(
+ Guice.createInjector(new WebServletModule()));
}
public TestRMWebServicesFairScheduler() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class)
- .contextPath("jersey-guice-filter").servletPath("/").build());
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
}
-
+
@Test
- public void testClusterScheduler() throws JSONException {
+ public void testClusterScheduler() throws JSONException, Exception {
WebResource r = resource();
- ClientResponse response =
- r.path("ws").path("v1").path("cluster").path("scheduler")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
@@ -103,51 +99,52 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
@Test
- public void testClusterSchedulerSlash() throws JSONException {
+ public void testClusterSchedulerSlash() throws JSONException, Exception {
WebResource r = resource();
- ClientResponse response =
- r.path("ws").path("v1").path("cluster").path("scheduler/")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler/").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
verifyClusterScheduler(json);
}
-
+
@Test
- public void testClusterSchedulerWithSubQueues()
- throws JSONException {
- FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+ public void testClusterSchedulerWithSubQueues() throws JSONException,
+ Exception {
+ FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueue
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
WebResource r = resource();
- ClientResponse response =
- r.path("ws").path("v1").path("cluster").path("scheduler")
- .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
JSONArray subQueueInfo = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
- .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
- .getJSONObject("childQueues").getJSONArray("queue");
+ .getJSONObject("childQueues").getJSONArray("queue")
+ .getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
// subQueueInfo is consist of subqueue1 and subqueue2 info
assertEquals(2, subQueueInfo.length());
// Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
try {
subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
- fail("FairSchedulerQueueInfo should omit field 'childQueues'"
- + "if child queue is empty.");
+ fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
+ "if child queue is empty.");
} catch (JSONException je) {
assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
}
}
- private void verifyClusterScheduler(JSONObject json) throws JSONException {
+ private void verifyClusterScheduler(JSONObject json) throws JSONException,
+ Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
index 40cf483..1e61186 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
@@ -457,7 +457,7 @@ public class TestRMWebServicesSchedulerActivities
if (object.getClass() == JSONObject.class) {
assertEquals("Number of allocations is wrong", 1, realValue);
} else if (object.getClass() == JSONArray.class) {
- assertEquals("Number of allocations is wrong in: " + object,
+ assertEquals("Number of allocations is wrong",
((JSONArray) object).length(), realValue);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39ad9890/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
deleted file mode 100644
index bb1fce0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.LocalConfigurationProvider;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import static java.util.stream.Collectors.toList;
-
-/**
- * This class can generate an XML configuration file of custom resource types.
- * See createInitialResourceTypes for the default values. All custom resource
- * type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the
- * getConfigurationInputStream method to get an InputStream of the XML. If you
- * want to have different number of resources in your tests, please see usages
- * of this class in this test class:
- * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}
- *
- */
-public class CustomResourceTypesConfigurationProvider
- extends LocalConfigurationProvider {
-
- private static class CustomResourceTypes {
- private int count;
- private String xml;
-
- CustomResourceTypes(String xml, int count) {
- this.xml = xml;
- this.count = count;
- }
-
- public int getCount() {
- return count;
- }
-
- public String getXml() {
- return xml;
- }
- }
-
- private static final String CUSTOM_RESOURCE_PREFIX = "customResource-";
-
- private static CustomResourceTypes customResourceTypes =
- createInitialResourceTypes();
-
- private static CustomResourceTypes createInitialResourceTypes() {
- return createCustomResourceTypes(2);
- }
-
- private static CustomResourceTypes createCustomResourceTypes(int count) {
- List<String> resourceTypeNames = generateResourceTypeNames(count);
-
- List<String> resourceUnitXmlElements = IntStream.range(0, count)
- .boxed()
- .map(i -> getResourceUnitsXml(resourceTypeNames.get(i)))
- .collect(toList());
-
- StringBuilder sb = new StringBuilder("<configuration>\n");
- sb.append(getResourceTypesXml(resourceTypeNames));
-
- for (String resourceUnitXml : resourceUnitXmlElements) {
- sb.append(resourceUnitXml);
-
- }
- sb.append("</configuration>");
-
- return new CustomResourceTypes(sb.toString(), count);
- }
-
- private static List<String> generateResourceTypeNames(int count) {
- return IntStream.range(0, count)
- .boxed()
- .map(i -> CUSTOM_RESOURCE_PREFIX + i)
- .collect(toList());
- }
-
- private static String getResourceUnitsXml(String resource) {
- return "<property>\n" + "<name>yarn.resource-types." + resource
- + ".units</name>\n" + "<value>k</value>\n" + "</property>\n";
- }
-
- private static String getResourceTypesXml(List<String> resources) {
- final String resourceTypes = makeCommaSeparatedString(resources);
-
- return "<property>\n" + "<name>yarn.resource-types</name>\n" + "<value>"
- + resourceTypes + "</value>\n" + "</property>\n";
- }
-
- private static String makeCommaSeparatedString(List<String> resources) {
- return resources.stream().collect(Collectors.joining(","));
- }
-
- @Override
- public InputStream getConfigurationInputStream(Configuration bootstrapConf,
- String name) throws YarnException, IOException {
- if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
- return new ByteArrayInputStream(
- customResourceTypes.getXml().getBytes());
- } else {
- return super.getConfigurationInputStream(bootstrapConf, name);
- }
- }
-
- public static void reset() {
- customResourceTypes = createInitialResourceTypes();
- }
-
- public static void setNumberOfResourceTypes(int count) {
- customResourceTypes = createCustomResourceTypes(count);
- }
-
- public static List<String> getCustomResourceTypes() {
- return generateResourceTypeNames(customResourceTypes.getCount());
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org