You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by rk...@apache.org on 2018/07/05 17:54:50 UTC
[1/2] hadoop git commit: YARN-7451. Add missing tests to verify the
presence of custom resources of RM apps and scheduler webservice endpoints
(snemeth via rkanter)
Repository: hadoop
Updated Branches:
refs/heads/trunk 172624702 -> 99febe7fd
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
new file mode 100644
index 0000000..7c5b6db
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoXmlVerifications.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.w3c.dom.Element;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlFloat;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Contains all value verifications that are needed to verify {@link AppInfo}
+ * XML documents.
+ */
+public final class AppInfoXmlVerifications {
+
+ private AppInfoXmlVerifications() {
+ //utility class
+ }
+
+ /**
+ * Tests whether {@link AppInfo} representation object contains the required
+ * values as per defined in the specified app parameter.
+ * @param info
+ * @param app an RMApp instance that contains the required values
+ */
+ public static void verify(Element info, RMApp app) {
+ checkStringMatch("id", app.getApplicationId()
+ .toString(), getXmlString(info, "id"));
+ checkStringMatch("user", app.getUser(),
+ getXmlString(info, "user"));
+ checkStringMatch("name", app.getName(),
+ getXmlString(info, "name"));
+ checkStringMatch("applicationType",
+ app.getApplicationType(), getXmlString(info, "applicationType"));
+ checkStringMatch("queue", app.getQueue(),
+ getXmlString(info, "queue"));
+ assertEquals("priority doesn't match", 0, getXmlInt(info, "priority"));
+ checkStringMatch("state", app.getState().toString(),
+ getXmlString(info, "state"));
+ checkStringMatch("finalStatus", app
+ .getFinalApplicationStatus().toString(),
+ getXmlString(info, "finalStatus"));
+ assertEquals("progress doesn't match", 0, getXmlFloat(info, "progress"),
+ 0.0);
+ if ("UNASSIGNED".equals(getXmlString(info, "trackingUI"))) {
+ checkStringMatch("trackingUI", "UNASSIGNED",
+ getXmlString(info, "trackingUI"));
+ }
+ WebServicesTestUtils.checkStringEqual("diagnostics",
+ app.getDiagnostics().toString(), getXmlString(info, "diagnostics"));
+ assertEquals("clusterId doesn't match",
+ ResourceManager.getClusterTimeStamp(),
+ getXmlLong(info, "clusterId"));
+ assertEquals("startedTime doesn't match", app.getStartTime(),
+ getXmlLong(info, "startedTime"));
+ assertEquals("finishedTime doesn't match", app.getFinishTime(),
+ getXmlLong(info, "finishedTime"));
+ assertTrue("elapsed time not greater than 0",
+ getXmlLong(info, "elapsedTime") > 0);
+ checkStringMatch("amHostHttpAddress", app
+ .getCurrentAppAttempt().getMasterContainer()
+ .getNodeHttpAddress(),
+ getXmlString(info, "amHostHttpAddress"));
+ assertTrue("amContainerLogs doesn't match",
+ getXmlString(info, "amContainerLogs").startsWith("http://"));
+ assertTrue("amContainerLogs doesn't contain user info",
+ getXmlString(info, "amContainerLogs").endsWith("/" + app.getUser()));
+ assertEquals("allocatedMB doesn't match", 1024,
+ getXmlInt(info, "allocatedMB"));
+ assertEquals("allocatedVCores doesn't match", 1,
+ getXmlInt(info, "allocatedVCores"));
+ assertEquals("queueUsagePerc doesn't match", 50.0f,
+ getXmlFloat(info, "queueUsagePercentage"), 0.01f);
+ assertEquals("clusterUsagePerc doesn't match", 50.0f,
+ getXmlFloat(info, "clusterUsagePercentage"), 0.01f);
+ assertEquals("numContainers doesn't match", 1,
+ getXmlInt(info, "runningContainers"));
+ assertNotNull("preemptedResourceSecondsMap should not be null",
+ info.getElementsByTagName("preemptedResourceSecondsMap"));
+ assertEquals("preemptedResourceMB doesn't match", app
+ .getRMAppMetrics().getResourcePreempted().getMemorySize(),
+ getXmlInt(info, "preemptedResourceMB"));
+ assertEquals("preemptedResourceVCores doesn't match", app
+ .getRMAppMetrics().getResourcePreempted().getVirtualCores(),
+ getXmlInt(info, "preemptedResourceVCores"));
+ assertEquals("numNonAMContainerPreempted doesn't match", app
+ .getRMAppMetrics().getNumNonAMContainersPreempted(),
+ getXmlInt(info, "numNonAMContainerPreempted"));
+ assertEquals("numAMContainerPreempted doesn't match", app
+ .getRMAppMetrics().getNumAMContainersPreempted(),
+ getXmlInt(info, "numAMContainerPreempted"));
+ assertEquals("Log aggregation Status doesn't match", app
+ .getLogAggregationStatusForAppReport().toString(),
+ getXmlString(info, "logAggregationStatus"));
+ assertEquals("unmanagedApplication doesn't match", app
+ .getApplicationSubmissionContext().getUnmanagedAM(),
+ getXmlBoolean(info, "unmanagedApplication"));
+ assertEquals("unmanagedApplication doesn't match",
+ app.getApplicationSubmissionContext().getNodeLabelExpression(),
+ getXmlString(info, "appNodeLabelExpression"));
+ assertEquals("unmanagedApplication doesn't match",
+ app.getAMResourceRequests().get(0).getNodeLabelExpression(),
+ getXmlString(info, "amNodeLabelExpression"));
+ assertEquals("amRPCAddress",
+ AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
+ getXmlString(info, "amRPCAddress"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
new file mode 100644
index 0000000..a8990ca
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/BufferedClientResponse.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.UniformInterfaceException;
+
+import javax.ws.rs.core.MediaType;
+import java.io.IOException;
+
+/**
+ * This class is merely a wrapper for {@link ClientResponse}. Given that the
+ * entity input stream of {@link ClientResponse} can be read only once by
+ * default and for some tests it is convenient to read the input stream many
+ * times, this class hides the details of how to do that and prevents
+ * unnecessary code duplication in tests.
+ */
+public class BufferedClientResponse {
+ private ClientResponse response;
+
+ public BufferedClientResponse(ClientResponse response) {
+ response.bufferEntity();
+ this.response = response;
+ }
+
+ public <T> T getEntity(Class<T> clazz)
+ throws ClientHandlerException, UniformInterfaceException {
+ try {
+ response.getEntityInputStream().reset();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return response.getEntity(clazz);
+ }
+
+ public MediaType getType() {
+ return response.getType();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
new file mode 100644
index 0000000..9d6a111
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/JsonCustomResourceTypeTestcase.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.http.JettyUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.core.MediaType;
+
+import java.util.function.Consumer;
+
+import static org.junit.Assert.*;
+
+/**
+ * This class hides the implementation details of how to verify the structure of
+ * JSON responses. Tests should only provide the path of the
+ * {@link WebResource}, the response from the resource and
+ * the verifier Consumer to
+ * {@link JsonCustomResourceTypeTestcase#verify(Consumer)}. An instance of
+ * {@link JSONObject} will be passed to that consumer to be able to
+ * verify the response.
+ */
+public class JsonCustomResourceTypeTestcase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(JsonCustomResourceTypeTestcase.class);
+
+ private final WebResource path;
+ private final BufferedClientResponse response;
+ private final JSONObject parsedResponse;
+
+ public JsonCustomResourceTypeTestcase(WebResource path,
+ BufferedClientResponse response) {
+ this.path = path;
+ this.response = response;
+ this.parsedResponse = response.getEntity(JSONObject.class);
+ }
+
+ public void verify(Consumer<JSONObject> verifier) {
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ logResponse();
+
+ String responseStr = response.getEntity(String.class);
+ if (responseStr == null || responseStr.isEmpty()) {
+ throw new IllegalStateException("Response is null or empty!");
+ }
+ verifier.accept(parsedResponse);
+ }
+
+ private void logResponse() {
+ String responseStr = response.getEntity(String.class);
+ LOG.info("Raw response from service URL {}: {}", path.toString(),
+ responseStr);
+ LOG.info("Parsed response from service URL {}: {}", path.toString(),
+ parsedResponse);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
new file mode 100644
index 0000000..6e58a89
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsJsonVerifications.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Performs value verifications on
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
+ * objects against the values of {@link ResourceRequest}. With the help of the
+ * {@link Builder}, users can also make verifications of the custom resource
+ * types and its values.
+ */
+public class ResourceRequestsJsonVerifications {
+ private final ResourceRequest resourceRequest;
+ private final JSONObject requestInfo;
+ private final Map<String, Long> customResourceTypes;
+ private final List<String> expectedCustomResourceTypes;
+
+ ResourceRequestsJsonVerifications(Builder builder) {
+ this.resourceRequest = builder.resourceRequest;
+ this.requestInfo = builder.requestInfo;
+ this.customResourceTypes = builder.customResourceTypes;
+ this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
+ }
+
+ public static void verify(JSONObject requestInfo, ResourceRequest rr)
+ throws JSONException {
+ createDefaultBuilder(requestInfo, rr).build().verify();
+ }
+
+ public static void verifyWithCustomResourceTypes(JSONObject requestInfo,
+ ResourceRequest resourceRequest, List<String> expectedResourceTypes)
+ throws JSONException {
+
+ createDefaultBuilder(requestInfo, resourceRequest)
+ .withExpectedCustomResourceTypes(expectedResourceTypes)
+ .withCustomResourceTypes(
+ extractActualCustomResourceTypes(requestInfo, expectedResourceTypes))
+ .build().verify();
+ }
+
+ private static Builder createDefaultBuilder(JSONObject requestInfo,
+ ResourceRequest resourceRequest) {
+ return new ResourceRequestsJsonVerifications.Builder()
+ .withRequest(resourceRequest)
+ .withRequestInfoJson(requestInfo);
+ }
+
+ private static Map<String, Long> extractActualCustomResourceTypes(
+ JSONObject requestInfo, List<String> expectedResourceTypes)
+ throws JSONException {
+ JSONObject capability = requestInfo.getJSONObject("capability");
+ Map<String, Long> resourceAndValue =
+ extractCustomResorceTypeValues(capability, expectedResourceTypes);
+ Map.Entry<String, Long> resourceEntry =
+ resourceAndValue.entrySet().iterator().next();
+
+ assertTrue(
+ "Found resource type: " + resourceEntry.getKey()
+ + " is not in expected resource types: " + expectedResourceTypes,
+ expectedResourceTypes.contains(resourceEntry.getKey()));
+
+ return resourceAndValue;
+ }
+
+ private static Map<String, Long> extractCustomResorceTypeValues(
+ JSONObject capability, List<String> expectedResourceTypes)
+ throws JSONException {
+ assertTrue(
+ "resourceCategory does not have resourceInformations: " + capability,
+ capability.has("resourceInformations"));
+
+ JSONObject resourceInformations =
+ capability.getJSONObject("resourceInformations");
+ assertTrue(
+ "resourceInformations does not have resourceInformation object: "
+ + resourceInformations,
+ resourceInformations.has("resourceInformation"));
+ JSONArray customResources =
+ resourceInformations.getJSONArray("resourceInformation");
+
+ // customResources will include vcores / memory as well
+ assertEquals(
+ "Different number of custom resource types found than expected",
+ expectedResourceTypes.size(), customResources.length() - 2);
+
+ Map<String, Long> resourceValues = Maps.newHashMap();
+ for (int i = 0; i < customResources.length(); i++) {
+ JSONObject customResource = customResources.getJSONObject(i);
+ assertTrue("Resource type does not have name field: " + customResource,
+ customResource.has("name"));
+ assertTrue("Resource type does not have name resourceType field: "
+ + customResource, customResource.has("resourceType"));
+ assertTrue(
+ "Resource type does not have name units field: " + customResource,
+ customResource.has("units"));
+ assertTrue(
+ "Resource type does not have name value field: " + customResource,
+ customResource.has("value"));
+
+ String name = customResource.getString("name");
+ String unit = customResource.getString("units");
+ String resourceType = customResource.getString("resourceType");
+ Long value = customResource.getLong("value");
+
+ if (ResourceInformation.MEMORY_URI.equals(name)
+ || ResourceInformation.VCORES_URI.equals(name)) {
+ continue;
+ }
+
+ assertTrue("Custom resource type " + name + " not found",
+ expectedResourceTypes.contains(name));
+ assertEquals("k", unit);
+ assertEquals(ResourceTypes.COUNTABLE,
+ ResourceTypes.valueOf(resourceType));
+ assertNotNull("Custom resource value " + value + " is null!", value);
+ resourceValues.put(name, value);
+ }
+
+ return resourceValues;
+ }
+
+ private void verify() throws JSONException {
+ assertEquals("nodeLabelExpression doesn't match",
+ resourceRequest.getNodeLabelExpression(),
+ requestInfo.getString("nodeLabelExpression"));
+ assertEquals("numContainers doesn't match",
+ resourceRequest.getNumContainers(),
+ requestInfo.getInt("numContainers"));
+ assertEquals("relaxLocality doesn't match",
+ resourceRequest.getRelaxLocality(),
+ requestInfo.getBoolean("relaxLocality"));
+ assertEquals("priority does not match",
+ resourceRequest.getPriority().getPriority(),
+ requestInfo.getInt("priority"));
+ assertEquals("resourceName does not match",
+ resourceRequest.getResourceName(),
+ requestInfo.getString("resourceName"));
+ assertEquals("memory does not match",
+ resourceRequest.getCapability().getMemorySize(),
+ requestInfo.getJSONObject("capability").getLong("memory"));
+ assertEquals("vCores does not match",
+ resourceRequest.getCapability().getVirtualCores(),
+ requestInfo.getJSONObject("capability").getLong("vCores"));
+
+ verifyAtLeastOneCustomResourceIsSerialized();
+
+ JSONObject executionTypeRequest =
+ requestInfo.getJSONObject("executionTypeRequest");
+ assertEquals("executionType does not match",
+ resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
+ executionTypeRequest.getString("executionType"));
+ assertEquals("enforceExecutionType does not match",
+ resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
+ executionTypeRequest.getBoolean("enforceExecutionType"));
+ }
+
+ /**
+ * JSON serialization produces "invalid JSON" by default as maps are
+ * serialized like this:
+ * "customResources":{"entry":{"key":"customResource-1","value":"0"}}
+ * If the map has multiple keys then multiple entries will be serialized.
+ * Our json parser in tests cannot handle duplicates therefore only one
+ * custom resource will be in the parsed json. See:
+ * https://issues.apache.org/jira/browse/YARN-7505
+ */
+ private void verifyAtLeastOneCustomResourceIsSerialized() {
+ boolean resourceFound = false;
+ for (String expectedCustomResourceType : expectedCustomResourceTypes) {
+ if (customResourceTypes.containsKey(expectedCustomResourceType)) {
+ resourceFound = true;
+ Long resourceValue =
+ customResourceTypes.get(expectedCustomResourceType);
+ assertNotNull("Resource value should not be null!", resourceValue);
+ }
+ }
+ assertTrue("No custom resource type can be found in the response!",
+ resourceFound);
+ }
+
+ /**
+ * Builder class for {@link ResourceRequestsJsonVerifications}.
+ */
+ public static final class Builder {
+ private List<String> expectedCustomResourceTypes = Lists.newArrayList();
+ private Map<String, Long> customResourceTypes;
+ private ResourceRequest resourceRequest;
+ private JSONObject requestInfo;
+
+ Builder() {
+ }
+
+ public static Builder create() {
+ return new Builder();
+ }
+
+ Builder withExpectedCustomResourceTypes(
+ List<String> expectedCustomResourceTypes) {
+ this.expectedCustomResourceTypes = expectedCustomResourceTypes;
+ return this;
+ }
+
+ Builder withCustomResourceTypes(
+ Map<String, Long> customResourceTypes) {
+ this.customResourceTypes = customResourceTypes;
+ return this;
+ }
+
+ Builder withRequest(ResourceRequest resourceRequest) {
+ this.resourceRequest = resourceRequest;
+ return this;
+ }
+
+ Builder withRequestInfoJson(JSONObject requestInfo) {
+ this.requestInfo = requestInfo;
+ return this;
+ }
+
+ public ResourceRequestsJsonVerifications build() {
+ return new ResourceRequestsJsonVerifications(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
new file mode 100644
index 0000000..af9b0f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/ResourceRequestsXmlVerifications.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlBoolean;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlInt;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Performs value verifications on
+ * {@link org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceRequestInfo}
+ * objects against the values of {@link ResourceRequest}. With the help of the
+ * {@link Builder}, users can also make verifications of the custom resource
+ * types and its values.
+ */
+public class ResourceRequestsXmlVerifications {
+ private final ResourceRequest resourceRequest;
+ private final Element requestInfo;
+ private final Map<String, Long> customResourceTypes;
+ private final List<String> expectedCustomResourceTypes;
+
+ ResourceRequestsXmlVerifications(Builder builder) {
+ this.resourceRequest = builder.resourceRequest;
+ this.requestInfo = builder.requestInfo;
+ this.customResourceTypes = builder.customResourceTypes;
+ this.expectedCustomResourceTypes = builder.expectedCustomResourceTypes;
+ }
+
+ public static void verifyWithCustomResourceTypes(Element requestInfo,
+ ResourceRequest resourceRequest, List<String> expectedResourceTypes) {
+
+ createDefaultBuilder(requestInfo, resourceRequest)
+ .withExpectedCustomResourceTypes(expectedResourceTypes)
+ .withCustomResourceTypes(extractActualCustomResourceType(requestInfo,
+ expectedResourceTypes))
+ .build().verify();
+ }
+
+ private static Builder createDefaultBuilder(Element requestInfo,
+ ResourceRequest resourceRequest) {
+ return new ResourceRequestsXmlVerifications.Builder()
+ .withRequest(resourceRequest).withRequestInfo(requestInfo);
+ }
+
+ private static Map<String, Long> extractActualCustomResourceType(
+ Element requestInfo, List<String> expectedResourceTypes) {
+ Element capability =
+ (Element) requestInfo.getElementsByTagName("capability").item(0);
+
+ return extractCustomResorceTypes(capability,
+ Sets.newHashSet(expectedResourceTypes));
+ }
+
+ private static Map<String, Long> extractCustomResorceTypes(Element capability,
+ Set<String> expectedResourceTypes) {
+ assertEquals(
+ toXml(capability) + " should have only one resourceInformations child!",
+ 1, capability.getElementsByTagName("resourceInformations").getLength());
+ Element resourceInformations = (Element) capability
+ .getElementsByTagName("resourceInformations").item(0);
+
+ NodeList customResources =
+ resourceInformations.getElementsByTagName("resourceInformation");
+
+ // customResources will include vcores / memory as well
+ assertEquals(
+ "Different number of custom resource types found than expected",
+ expectedResourceTypes.size(), customResources.getLength() - 2);
+
+ Map<String, Long> resourceTypesAndValues = Maps.newHashMap();
+ for (int i = 0; i < customResources.getLength(); i++) {
+ Element customResource = (Element) customResources.item(i);
+ String name = getXmlString(customResource, "name");
+ String unit = getXmlString(customResource, "units");
+ String resourceType = getXmlString(customResource, "resourceType");
+ Long value = getXmlLong(customResource, "value");
+
+ if (ResourceInformation.MEMORY_URI.equals(name)
+ || ResourceInformation.VCORES_URI.equals(name)) {
+ continue;
+ }
+
+ assertTrue("Custom resource type " + name + " not found",
+ expectedResourceTypes.contains(name));
+ assertEquals("k", unit);
+ assertEquals(ResourceTypes.COUNTABLE,
+ ResourceTypes.valueOf(resourceType));
+ assertNotNull("Resource value should not be null for resource type "
+ + resourceType + ", listing xml contents: " + toXml(customResource),
+ value);
+ resourceTypesAndValues.put(name, value);
+ }
+
+ return resourceTypesAndValues;
+ }
+
+ private void verify() {
+ assertEquals("nodeLabelExpression doesn't match",
+ resourceRequest.getNodeLabelExpression(),
+ getXmlString(requestInfo, "nodeLabelExpression"));
+ assertEquals("numContainers doesn't match",
+ resourceRequest.getNumContainers(),
+ getXmlInt(requestInfo, "numContainers"));
+ assertEquals("relaxLocality doesn't match",
+ resourceRequest.getRelaxLocality(),
+ getXmlBoolean(requestInfo, "relaxLocality"));
+ assertEquals("priority does not match",
+ resourceRequest.getPriority().getPriority(),
+ getXmlInt(requestInfo, "priority"));
+ assertEquals("resourceName does not match",
+ resourceRequest.getResourceName(),
+ getXmlString(requestInfo, "resourceName"));
+ Element capability = (Element) requestInfo
+ .getElementsByTagName("capability").item(0);
+ assertEquals("memory does not match",
+ resourceRequest.getCapability().getMemorySize(),
+ getXmlLong(capability, "memory"));
+ assertEquals("vCores does not match",
+ resourceRequest.getCapability().getVirtualCores(),
+ getXmlLong(capability, "vCores"));
+
+ for (String expectedCustomResourceType : expectedCustomResourceTypes) {
+ assertTrue(
+ "Custom resource type " + expectedCustomResourceType
+ + " cannot be found!",
+ customResourceTypes.containsKey(expectedCustomResourceType));
+
+ Long resourceValue = customResourceTypes.get(expectedCustomResourceType);
+ assertNotNull("Resource value should not be null!", resourceValue);
+ }
+
+ Element executionTypeRequest = (Element) requestInfo
+ .getElementsByTagName("executionTypeRequest").item(0);
+ assertEquals("executionType does not match",
+ resourceRequest.getExecutionTypeRequest().getExecutionType().name(),
+ getXmlString(executionTypeRequest, "executionType"));
+ assertEquals("enforceExecutionType does not match",
+ resourceRequest.getExecutionTypeRequest().getEnforceExecutionType(),
+ getXmlBoolean(executionTypeRequest, "enforceExecutionType"));
+ }
+
+ /**
+ * Builder class for {@link ResourceRequestsXmlVerifications}.
+ */
+ public static final class Builder {
+ private List<String> expectedCustomResourceTypes = Lists.newArrayList();
+ private Map<String, Long> customResourceTypes;
+ private ResourceRequest resourceRequest;
+ private Element requestInfo;
+
+ Builder() {
+ }
+
+ public static Builder create() {
+ return new Builder();
+ }
+
+ Builder withExpectedCustomResourceTypes(
+ List<String> expectedCustomResourceTypes) {
+ this.expectedCustomResourceTypes = expectedCustomResourceTypes;
+ return this;
+ }
+
+ Builder withCustomResourceTypes(Map<String, Long> customResourceTypes) {
+ this.customResourceTypes = customResourceTypes;
+ return this;
+ }
+
+ Builder withRequest(ResourceRequest resourceRequest) {
+ this.resourceRequest = resourceRequest;
+ return this;
+ }
+
+ Builder withRequestInfo(Element requestInfo) {
+ this.requestInfo = requestInfo;
+ return this;
+ }
+
+ public ResourceRequestsXmlVerifications build() {
+ return new ResourceRequestsXmlVerifications(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
new file mode 100644
index 0000000..29260aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/XmlCustomResourceTypeTestCase.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import com.sun.jersey.api.client.WebResource;
+import org.apache.hadoop.http.JettyUtils;
+import org.codehaus.jettison.json.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Node;
+import org.xml.sax.InputSource;
+
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.*;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.util.function.Consumer;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class hides the implementation details of how to verify the structure of
+ * XML responses. Tests should only provide the path of the
+ * {@link WebResource}, the response from the resource and
+ * the verifier Consumer to
+ * {@link XmlCustomResourceTypeTestCase#verify(Consumer)}. An instance of
+ * {@link JSONObject} will be passed to that consumer to be able to
+ * verify the response.
+ */
+public class XmlCustomResourceTypeTestCase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(XmlCustomResourceTypeTestCase.class);
+
+ private WebResource path;
+ private BufferedClientResponse response;
+ private Document parsedResponse;
+
+ public XmlCustomResourceTypeTestCase(WebResource path,
+ BufferedClientResponse response) {
+ this.path = path;
+ this.response = response;
+ }
+
+ public void verify(Consumer<Document> verifier) {
+ assertEquals(MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ parsedResponse = parseXml(response);
+ logResponse(parsedResponse);
+ verifier.accept(parsedResponse);
+ }
+
+ private Document parseXml(BufferedClientResponse response) {
+ try {
+ String xml = response.getEntity(String.class);
+ DocumentBuilder db =
+ DocumentBuilderFactory.newInstance().newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(xml));
+
+ return db.parse(is);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void logResponse(Document doc) {
+ String responseStr = response.getEntity(String.class);
+ LOG.info("Raw response from service URL {}: {}", path.toString(),
+ responseStr);
+ LOG.info("Parsed response from service URL {}: {}", path.toString(),
+ toXml(doc));
+ }
+
+ public static String toXml(Node node) {
+ StringWriter writer;
+ try {
+ TransformerFactory tf = TransformerFactory.newInstance();
+ Transformer transformer = tf.newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.setOutputProperty(
+ "{http://xml.apache.org/xslt}indent" + "-amount", "2");
+ writer = new StringWriter();
+ transformer.transform(new DOMSource(node), new StreamResult(writer));
+ } catch (TransformerException e) {
+ throw new RuntimeException(e);
+ }
+
+ return writer.getBuffer().toString();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-7451. Add missing tests to verify the
presence of custom resources of RM apps and scheduler webservice endpoints
(snemeth via rkanter)
Posted by rk...@apache.org.
YARN-7451. Add missing tests to verify the presence of custom resources of RM apps and scheduler webservice endpoints (snemeth via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99febe7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99febe7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99febe7f
Branch: refs/heads/trunk
Commit: 99febe7fd50c31c0f5dd40fa7f376f2c1f64f8c3
Parents: 1726247
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Jul 5 10:54:19 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Jul 5 10:54:19 2018 -0700
----------------------------------------------------------------------
.../resourcemanager/webapp/dao/AppInfo.java | 2 +-
.../webapp/dao/SchedulerInfo.java | 8 +-
.../fair/TestFairSchedulerConfiguration.java | 9 +-
.../webapp/TestRMWebServices.java | 31 ++-
.../webapp/TestRMWebServicesApps.java | 14 +-
...estRMWebServicesAppsCustomResourceTypes.java | 242 +++++++++++++++++
.../webapp/TestRMWebServicesCapacitySched.java | 30 +-
.../TestRMWebServicesConfigurationMutation.java | 5 +
.../webapp/TestRMWebServicesFairScheduler.java | 95 +++----
.../TestRMWebServicesSchedulerActivities.java | 2 +-
...ustomResourceTypesConfigurationProvider.java | 138 ++++++++++
.../FairSchedulerJsonVerifications.java | 139 ++++++++++
.../FairSchedulerXmlVerifications.java | 153 +++++++++++
...ervicesFairSchedulerCustomResourceTypes.java | 271 +++++++++++++++++++
.../webapp/helper/AppInfoJsonVerifications.java | 123 +++++++++
.../webapp/helper/AppInfoXmlVerifications.java | 132 +++++++++
.../webapp/helper/BufferedClientResponse.java | 57 ++++
.../helper/JsonCustomResourceTypeTestcase.java | 77 ++++++
.../ResourceRequestsJsonVerifications.java | 252 +++++++++++++++++
.../ResourceRequestsXmlVerifications.java | 215 +++++++++++++++
.../helper/XmlCustomResourceTypeTestCase.java | 112 ++++++++
21 files changed, 2020 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index d47f13d..9d82bc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -479,7 +479,7 @@ public class AppInfo {
public int getNumNonAMContainersPreempted() {
return numNonAMContainerPreempted;
}
-
+
public int getNumAMContainersPreempted() {
return numAMContainerPreempted;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 81491b1..163f707 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -41,8 +41,9 @@ public class SchedulerInfo {
protected EnumSet<SchedulerResourceTypes> schedulingResourceTypes;
protected int maximumClusterPriority;
+ // JAXB needs this
public SchedulerInfo() {
- } // JAXB needs this
+ }
public SchedulerInfo(final ResourceManager rm) {
ResourceScheduler rs = rm.getResourceScheduler();
@@ -74,7 +75,10 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
- return Arrays.toString(minAllocResource.getResource().getResources());
+ if (minAllocResource != null) {
+ return Arrays.toString(minAllocResource.getResource().getResources());
+ }
+ return null;
}
public int getMaxClusterLevelAppPriority() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
index 76a5af5..70f83ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerConfiguration.java
@@ -48,6 +48,9 @@ import org.apache.log4j.spi.LoggingEvent;
import org.junit.Assert;
import org.junit.Test;
+/**
+ * Tests fair scheduler configuration.
+ */
public class TestFairSchedulerConfiguration {
private static final String A_CUSTOM_RESOURCE = "a-custom-resource";
@@ -242,12 +245,12 @@ public class TestFairSchedulerConfiguration {
parseResourceConfigValue(" vcores = 75 % , memory-mb = 40 % , "
+ "test1 = 50 % ").getResource(clusterResource));
}
-
+
@Test(expected = AllocationConfigurationException.class)
public void testNoUnits() throws Exception {
parseResourceConfigValue("1024");
}
-
+
@Test(expected = AllocationConfigurationException.class)
public void testOnlyMemory() throws Exception {
parseResourceConfigValue("1024mb");
@@ -257,7 +260,7 @@ public class TestFairSchedulerConfiguration {
public void testOnlyCPU() throws Exception {
parseResourceConfigValue("1024vcores");
}
-
+
@Test(expected = AllocationConfigurationException.class)
public void testGibberish() throws Exception {
parseResourceConfigValue("1o24vc0res");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 0702d65..3902889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -53,11 +53,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
-import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.*;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -76,11 +72,12 @@ import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
-import org.eclipse.jetty.server.Response;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
@@ -96,6 +93,8 @@ import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServices extends JerseyTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestRMWebServices.class);
private static MockRM rm;
@@ -472,19 +471,19 @@ public class TestRMWebServices extends JerseyTestBase {
QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
- long totalMBExpect =
+ long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
- long totalVirtualCoresExpect =
+ long totalVirtualCoresExpect =
metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
- assertEquals("appsSubmitted doesn't match",
+ assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
- assertEquals("appsCompleted doesn't match",
+ assertEquals("appsCompleted doesn't match",
metrics.getAppsCompleted(), completedApps);
assertEquals("reservedMB doesn't match",
metrics.getReservedMB(), reservedMB);
- assertEquals("availableMB doesn't match",
+ assertEquals("availableMB doesn't match",
metrics.getAvailableMB(), availableMB);
- assertEquals("allocatedMB doesn't match",
+ assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
assertEquals("reservedVirtualCores doesn't match",
metrics.getReservedVirtualCores(), reservedVirtualCores);
@@ -597,11 +596,13 @@ public class TestRMWebServices extends JerseyTestBase {
public void verifyClusterSchedulerFifo(JSONObject json) throws JSONException,
Exception {
- assertEquals("incorrect number of elements", 1, json.length());
+ assertEquals("incorrect number of elements in: " + json, 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
- assertEquals("incorrect number of elements", 1, info.length());
+ assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements", 11, info.length());
+
+ LOG.debug("schedulerInfo: {}", info);
+ assertEquals("incorrect number of elements in: " + info, 11, info.length());
verifyClusterSchedulerFifoGeneric(info.getString("type"),
info.getString("qstate"), (float) info.getDouble("capacity"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 6c6f400..15f94e1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -79,7 +79,7 @@ import com.sun.jersey.test.framework.WebAppDescriptor;
public class TestRMWebServicesApps extends JerseyTestBase {
private static MockRM rm;
-
+
private static final int CONTAINER_MB = 1024;
private static class WebServletModule extends ServletModule {
@@ -324,7 +324,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
- assertTrue("both app states of ACCEPTED and KILLED are not present",
+ assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
@@ -375,12 +375,12 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("incorrect number of elements", 1, apps.length());
array = apps.getJSONArray("app");
assertEquals("incorrect number of elements", 2, array.length());
- assertTrue("both app states of ACCEPTED and KILLED are not present",
+ assertTrue("both app states of ACCEPTED and KILLED are not present",
(array.getJSONObject(0).getString("state").equals("ACCEPTED") &&
array.getJSONObject(1).getString("state").equals("KILLED")) ||
(array.getJSONObject(0).getString("state").equals("KILLED") &&
array.getJSONObject(1).getString("state").equals("ACCEPTED")));
-
+
rm.stop();
}
@@ -511,7 +511,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("apps").queryParam("finalStatus", FinalApplicationStatus.UNDEFINED.toString())
+ .path("apps").queryParam("finalStatus",
+ FinalApplicationStatus.UNDEFINED.toString())
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
@@ -1804,7 +1805,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
int numAttempt = 1;
while (true) {
// fail the AM by sending CONTAINER_FINISHED event without registering.
- amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1,
+ ContainerState.COMPLETE);
rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.FAILED);
if (numAttempt == maxAppAttempts) {
rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
new file mode 100644
index 0000000..83e0056
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsCustomResourceTypes.java
@@ -0,0 +1,242 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler.CustomResourceTypesConfigurationProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoJsonVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.AppInfoXmlVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.BufferedClientResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.JsonCustomResourceTypeTestcase;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsJsonVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.ResourceRequestsXmlVerifications;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This test verifies that custom resource types are correctly serialized to XML
+ * and JSON when HTTP GET request is sent to the resource: ws/v1/cluster/apps.
+ */
+public class TestRMWebServicesAppsCustomResourceTypes extends JerseyTestBase {
+
+ private static MockRM rm;
+ private static final int CONTAINER_MB = 1024;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
+ initResourceTypes(conf);
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+
+ private void initResourceTypes(Configuration conf) {
+ conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ CustomResourceTypesConfigurationProvider.class.getName());
+ ResourceUtils.resetResourceTypes(conf);
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ createInjectorForWebServletModule();
+ }
+
+ private void createInjectorForWebServletModule() {
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
+ }
+
+ public TestRMWebServicesAppsCustomResourceTypes() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testRunningAppXml() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+ am1.allocate("*", 2048, 1, new ArrayList<>());
+ amNodeManager.nodeHeartbeat(true);
+
+ WebResource r = resource();
+ WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ XmlCustomResourceTypeTestCase testCase =
+ new XmlCustomResourceTypeTestCase(path,
+ new BufferedClientResponse(response));
+ testCase.verify(document -> {
+ NodeList apps = document.getElementsByTagName("apps");
+ assertEquals("incorrect number of apps elements", 1, apps.getLength());
+
+ NodeList appArray = ((Element)(apps.item(0)))
+ .getElementsByTagName("app");
+ assertEquals("incorrect number of app elements", 1, appArray.getLength());
+
+ verifyAppsXML(appArray, app1);
+ });
+
+ rm.stop();
+ }
+
+ @Test
+ public void testRunningAppJson() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
+ am1.allocate("*", 2048, 1, new ArrayList<>());
+ amNodeManager.nodeHeartbeat(true);
+
+ WebResource r = resource();
+ WebResource path = r.path("ws").path("v1").path("cluster").path("apps");
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ JsonCustomResourceTypeTestcase testCase =
+ new JsonCustomResourceTypeTestcase(path,
+ new BufferedClientResponse(response));
+ testCase.verify(json -> {
+ try {
+ assertEquals("incorrect number of apps elements", 1, json.length());
+ JSONObject apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of app elements", 1, apps.length());
+ JSONArray array = apps.getJSONArray("app");
+ assertEquals("incorrect count of app", 1, array.length());
+
+ verifyAppInfoJson(array.getJSONObject(0), app1);
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ });
+
+ rm.stop();
+ }
+
+ private void verifyAppsXML(NodeList appArray, RMApp app) {
+ for (int i = 0; i < appArray.getLength(); i++) {
+ Element element = (Element) appArray.item(i);
+ AppInfoXmlVerifications.verify(element, app);
+
+ NodeList resourceRequests =
+ element.getElementsByTagName("resourceRequests");
+ assertEquals(1, resourceRequests.getLength());
+ Node resourceRequest = resourceRequests.item(0);
+ ResourceRequest rr =
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getApplicationAttempt(
+ app.getCurrentAppAttempt().getAppAttemptId())
+ .getAppSchedulingInfo().getAllResourceRequests().get(0);
+ ResourceRequestsXmlVerifications.verifyWithCustomResourceTypes(
+ (Element) resourceRequest, rr,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+ }
+
+ private void verifyAppInfoJson(JSONObject info, RMApp app) throws
+ JSONException {
+ int expectedNumberOfElements = getExpectedNumberOfElements(app);
+
+ assertEquals("incorrect number of elements", expectedNumberOfElements,
+ info.length());
+
+ AppInfoJsonVerifications.verify(info, app);
+
+ JSONArray resourceRequests = info.getJSONArray("resourceRequests");
+ JSONObject requestInfo = resourceRequests.getJSONObject(0);
+ ResourceRequest rr =
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getApplicationAttempt(app.getCurrentAppAttempt().getAppAttemptId())
+ .getAppSchedulingInfo().getAllResourceRequests().get(0);
+
+ ResourceRequestsJsonVerifications.verifyWithCustomResourceTypes(
+ requestInfo, rr,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+
+ private int getExpectedNumberOfElements(RMApp app) {
+ int expectedNumberOfElements = 40 + 2; // 2 -> resourceRequests
+ if (app.getApplicationSubmissionContext()
+ .getNodeLabelExpression() != null) {
+ expectedNumberOfElements++;
+ }
+
+ if (app.getAMResourceRequests().get(0).getNodeLabelExpression() != null) {
+ expectedNumberOfElements++;
+ }
+
+ if (AppInfo
+ .getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()) != null) {
+ expectedNumberOfElements++;
+ }
+ return expectedNumberOfElements;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index e37f76f..46d0a66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -146,7 +146,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
config.setUserLimitFactor(B2, 100.0f);
config.setCapacity(B3, 0.5f);
config.setUserLimitFactor(B3, 100.0f);
-
+
config.setQueues(A1, new String[] {"a1a", "a1b"});
final String A1A = A1 + ".a1a";
config.setCapacity(A1A, 85);
@@ -254,7 +254,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
}
}
- public void verifySubQueueXML(Element qElem, String q,
+ public void verifySubQueueXML(Element qElem, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws Exception {
NodeList children = qElem.getChildNodes();
@@ -317,30 +317,34 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifyClusterScheduler(JSONObject json) throws JSONException,
Exception {
- assertEquals("incorrect number of elements", 1, json.length());
+ assertEquals("incorrect number of elements in: " + json, 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
- assertEquals("incorrect number of elements", 1, info.length());
+ assertEquals("incorrect number of elements in: " + info, 1, info.length());
info = info.getJSONObject("schedulerInfo");
- assertEquals("incorrect number of elements", 8, info.length());
+ assertEquals("incorrect number of elements in: " + info, 8, info.length());
verifyClusterSchedulerGeneric(info.getString("type"),
(float) info.getDouble("usedCapacity"),
(float) info.getDouble("capacity"),
(float) info.getDouble("maxCapacity"), info.getString("queueName"));
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
- assertEquals("incorrect number of elements", 3, health.length());
+ assertEquals("incorrect number of elements in: " + health, 3,
+ health.length());
JSONArray operationsInfo = health.getJSONArray("operationsInfo");
- assertEquals("incorrect number of elements", 4, operationsInfo.length());
+ assertEquals("incorrect number of elements in: " + health, 4,
+ operationsInfo.length());
JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
- assertEquals("incorrect number of elements", 3, lastRunDetails.length());
+ assertEquals("incorrect number of elements in: " + health, 3,
+ lastRunDetails.length());
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
- assertEquals("incorrect number of elements", 2, arr.length());
+ assertEquals("incorrect number of elements in: " + arr, 2, arr.length());
// test subqueues
for (int i = 0; i < arr.length(); i++) {
JSONObject obj = arr.getJSONObject(i);
- String q = CapacitySchedulerConfiguration.ROOT + "." + obj.getString("queueName");
+ String q = CapacitySchedulerConfiguration.ROOT + "." +
+ obj.getString("queueName");
verifySubQueue(obj, q, 100, 100);
}
}
@@ -355,7 +359,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
assertTrue("queueName doesn't match", "root".matches(queueName));
}
- private void verifySubQueue(JSONObject info, String q,
+ private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
int numExpectedElements = 20;
@@ -464,7 +468,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
csConf.getUserLimitFactor(q), info.userLimitFactor, 1e-3f);
}
- //Return a child Node of node with the tagname or null if none exists
+ //Return a child Node of node with the tagname or null if none exists
private Node getChildNodeByName(Node node, String tagname) {
NodeList nodeList = node.getChildNodes();
for (int i=0; i < nodeList.getLength(); ++i) {
@@ -514,7 +518,7 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
for (int j=0; j<users.getLength(); ++j) {
Node user = users.item(j);
String username = getChildNodeByName(user, "username")
- .getTextContent();
+ .getTextContent();
assertTrue(username.equals("user1") || username.equals("user2"));
//Should be a parsable integer
Integer.parseInt(getChildNodeByName(getChildNodeByName(user,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 3d28f12..99b5648 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -42,6 +42,8 @@ import org.apache.hadoop.yarn.webapp.util.YarnWebServiceUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response.Status;
@@ -59,6 +61,8 @@ import static org.junit.Assert.assertNull;
* Test scheduler configuration mutation via REST API.
*/
public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(TestRMWebServicesConfigurationMutation.class);
private static final File CONF_FILE = new File(new File("target",
"test-classes"), YarnConfiguration.CS_CONFIGURATION_FILE);
@@ -396,6 +400,7 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
.entity(YarnWebServiceUtils.toJson(updateInfo,
SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
.put(ClientResponse.class);
+ LOG.debug("Response headers: " + response.getHeaders());
assertEquals(Status.OK.getStatusCode(), response.getStatus());
CapacitySchedulerConfiguration newCSConf = cs.getConfiguration();
assertEquals(0.2f, newCSConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
index e77785b..58c72ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesFairScheduler.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,13 +16,14 @@
* limitations under the License.
*/
-package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-import javax.ws.rs.core.MediaType;
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -30,6 +31,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
+
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
@@ -38,18 +42,18 @@ import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Before;
import org.junit.Test;
+import javax.ws.rs.core.MediaType;
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+/**
+ * Tests RM Webservices fair scheduler resources.
+ */
public class TestRMWebServicesFairScheduler extends JerseyTestBase {
private static MockRM rm;
private static YarnConfiguration conf;
-
+
private static class WebServletModule extends ServletModule {
@Override
protected void configureServlets() {
@@ -58,7 +62,7 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
bind(GenericExceptionHandler.class);
conf = new YarnConfiguration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
- ResourceScheduler.class);
+ ResourceScheduler.class);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
serve("/*").with(GuiceContainer.class);
@@ -66,32 +70,32 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
static {
- GuiceServletConfig.setInjector(
- Guice.createInjector(new WebServletModule()));
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- GuiceServletConfig.setInjector(
- Guice.createInjector(new WebServletModule()));
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
}
public TestRMWebServicesFairScheduler() {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class)
- .contextPath("jersey-guice-filter").servletPath("/").build());
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
}
-
+
@Test
- public void testClusterScheduler() throws JSONException, Exception {
+ public void testClusterScheduler() throws JSONException {
WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("scheduler").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster").path("scheduler")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
@@ -99,52 +103,51 @@ public class TestRMWebServicesFairScheduler extends JerseyTestBase {
}
@Test
- public void testClusterSchedulerSlash() throws JSONException, Exception {
+ public void testClusterSchedulerSlash() throws JSONException {
WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("scheduler/").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster").path("scheduler/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
verifyClusterScheduler(json);
}
-
+
@Test
- public void testClusterSchedulerWithSubQueues() throws JSONException,
- Exception {
- FairScheduler scheduler = (FairScheduler)rm.getResourceScheduler();
+ public void testClusterSchedulerWithSubQueues()
+ throws JSONException {
+ FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
QueueManager queueManager = scheduler.getQueueManager();
// create LeafQueue
queueManager.getLeafQueue("root.q.subqueue1", true);
queueManager.getLeafQueue("root.q.subqueue2", true);
WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("scheduler").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster").path("scheduler")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
response.getType().toString());
JSONObject json = response.getEntity(JSONObject.class);
JSONArray subQueueInfo = json.getJSONObject("scheduler")
.getJSONObject("schedulerInfo").getJSONObject("rootQueue")
- .getJSONObject("childQueues").getJSONArray("queue")
- .getJSONObject(1).getJSONObject("childQueues").getJSONArray("queue");
+ .getJSONObject("childQueues").getJSONArray("queue").getJSONObject(1)
+ .getJSONObject("childQueues").getJSONArray("queue");
// subQueueInfo is consist of subqueue1 and subqueue2 info
assertEquals(2, subQueueInfo.length());
// Verify 'childQueues' field is omitted from FairSchedulerLeafQueueInfo.
try {
subQueueInfo.getJSONObject(1).getJSONObject("childQueues");
- fail("FairSchedulerQueueInfo should omit field 'childQueues'" +
- "if child queue is empty.");
+ fail("FairSchedulerQueueInfo should omit field 'childQueues'"
+ + "if child queue is empty.");
} catch (JSONException je) {
assertEquals("JSONObject[\"childQueues\"] not found.", je.getMessage());
}
}
- private void verifyClusterScheduler(JSONObject json) throws JSONException,
- Exception {
+ private void verifyClusterScheduler(JSONObject json) throws JSONException {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("scheduler");
assertEquals("incorrect number of elements", 1, info.length());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
index 1e61186..40cf483 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java
@@ -457,7 +457,7 @@ public class TestRMWebServicesSchedulerActivities
if (object.getClass() == JSONObject.class) {
assertEquals("Number of allocations is wrong", 1, realValue);
} else if (object.getClass() == JSONArray.class) {
- assertEquals("Number of allocations is wrong",
+ assertEquals("Number of allocations is wrong in: " + object,
((JSONArray) object).length(), realValue);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
new file mode 100644
index 0000000..bb1fce0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/CustomResourceTypesConfigurationProvider.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import static java.util.stream.Collectors.toList;
+
+/**
+ * This class can generate an XML configuration file of custom resource types.
+ * See createInitialResourceTypes for the default values. All custom resource
+ * type is prefixed with CUSTOM_RESOURCE_PREFIX. Please use the
+ * getConfigurationInputStream method to get an InputStream of the XML. If you
+ * want to have different number of resources in your tests, please see usages
+ * of this class in this test class:
+ * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}
+ *
+ */
+public class CustomResourceTypesConfigurationProvider
+ extends LocalConfigurationProvider {
+
+ private static class CustomResourceTypes {
+ private int count;
+ private String xml;
+
+ CustomResourceTypes(String xml, int count) {
+ this.xml = xml;
+ this.count = count;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+ public String getXml() {
+ return xml;
+ }
+ }
+
+ private static final String CUSTOM_RESOURCE_PREFIX = "customResource-";
+
+ private static CustomResourceTypes customResourceTypes =
+ createInitialResourceTypes();
+
+ private static CustomResourceTypes createInitialResourceTypes() {
+ return createCustomResourceTypes(2);
+ }
+
+ private static CustomResourceTypes createCustomResourceTypes(int count) {
+ List<String> resourceTypeNames = generateResourceTypeNames(count);
+
+ List<String> resourceUnitXmlElements = IntStream.range(0, count)
+ .boxed()
+ .map(i -> getResourceUnitsXml(resourceTypeNames.get(i)))
+ .collect(toList());
+
+ StringBuilder sb = new StringBuilder("<configuration>\n");
+ sb.append(getResourceTypesXml(resourceTypeNames));
+
+ for (String resourceUnitXml : resourceUnitXmlElements) {
+ sb.append(resourceUnitXml);
+
+ }
+ sb.append("</configuration>");
+
+ return new CustomResourceTypes(sb.toString(), count);
+ }
+
+ private static List<String> generateResourceTypeNames(int count) {
+ return IntStream.range(0, count)
+ .boxed()
+ .map(i -> CUSTOM_RESOURCE_PREFIX + i)
+ .collect(toList());
+ }
+
+ private static String getResourceUnitsXml(String resource) {
+ return "<property>\n" + "<name>yarn.resource-types." + resource
+ + ".units</name>\n" + "<value>k</value>\n" + "</property>\n";
+ }
+
+ private static String getResourceTypesXml(List<String> resources) {
+ final String resourceTypes = makeCommaSeparatedString(resources);
+
+ return "<property>\n" + "<name>yarn.resource-types</name>\n" + "<value>"
+ + resourceTypes + "</value>\n" + "</property>\n";
+ }
+
+ private static String makeCommaSeparatedString(List<String> resources) {
+ return resources.stream().collect(Collectors.joining(","));
+ }
+
+ @Override
+ public InputStream getConfigurationInputStream(Configuration bootstrapConf,
+ String name) throws YarnException, IOException {
+ if (YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE.equals(name)) {
+ return new ByteArrayInputStream(
+ customResourceTypes.getXml().getBytes());
+ } else {
+ return super.getConfigurationInputStream(bootstrapConf, name);
+ }
+ }
+
+ public static void reset() {
+ customResourceTypes = createInitialResourceTypes();
+ }
+
+ public static void setNumberOfResourceTypes(int count) {
+ customResourceTypes = createCustomResourceTypes(count);
+ }
+
+ public static List<String> getCustomResourceTypes() {
+ return generateResourceTypeNames(customResourceTypes.getCount());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
new file mode 100644
index 0000000..924411a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerJsonVerifications.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test helper class is primarily used by
+ * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
+ */
+public class FairSchedulerJsonVerifications {
+
+ private static final Set<String> RESOURCE_FIELDS =
+ Sets.newHashSet("minResources", "amUsedResources", "amMaxResources",
+ "fairResources", "clusterResources", "reservedResources",
+ "maxResources", "usedResources", "steadyFairResources",
+ "demandResources");
+ private final Set<String> customResourceTypes;
+
+ FairSchedulerJsonVerifications(List<String> customResourceTypes) {
+ this.customResourceTypes = Sets.newHashSet(customResourceTypes);
+ }
+
+ public void verify(JSONObject jsonObject) {
+ try {
+ verifyResourcesContainDefaultResourceTypes(jsonObject, RESOURCE_FIELDS);
+ verifyResourcesContainCustomResourceTypes(jsonObject, RESOURCE_FIELDS);
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void verifyResourcesContainDefaultResourceTypes(JSONObject queue,
+ Set<String> resourceCategories) throws JSONException {
+ for (String resourceCategory : resourceCategories) {
+ boolean hasResourceCategory = queue.has(resourceCategory);
+ assertTrue("Queue " + queue + " does not have resource category key: "
+ + resourceCategory, hasResourceCategory);
+ verifyResourceContainsDefaultResourceTypes(
+ queue.getJSONObject(resourceCategory));
+ }
+ }
+
+ private void verifyResourceContainsDefaultResourceTypes(
+ JSONObject jsonObject) {
+ Object memory = jsonObject.opt("memory");
+ Object vCores = jsonObject.opt("vCores");
+
+ assertNotNull("Key 'memory' not found in: " + jsonObject, memory);
+ assertNotNull("Key 'vCores' not found in: " + jsonObject, vCores);
+ }
+
+ private void verifyResourcesContainCustomResourceTypes(JSONObject queue,
+ Set<String> resourceCategories) throws JSONException {
+ for (String resourceCategory : resourceCategories) {
+ assertTrue("Queue " + queue + " does not have resource category key: "
+ + resourceCategory, queue.has(resourceCategory));
+ verifyResourceContainsAllCustomResourceTypes(
+ queue.getJSONObject(resourceCategory));
+ }
+ }
+
+ private void verifyResourceContainsAllCustomResourceTypes(
+ JSONObject resourceCategory) throws JSONException {
+ assertTrue("resourceCategory does not have resourceInformations: "
+ + resourceCategory, resourceCategory.has("resourceInformations"));
+
+ JSONObject resourceInformations =
+ resourceCategory.getJSONObject("resourceInformations");
+ assertTrue(
+ "resourceInformations does not have resourceInformation object: "
+ + resourceInformations,
+ resourceInformations.has("resourceInformation"));
+ JSONArray customResources =
+ resourceInformations.getJSONArray("resourceInformation");
+
+ // customResources will include vcores / memory as well
+ assertEquals(
+ "Different number of custom resource types found than expected",
+ customResourceTypes.size(), customResources.length() - 2);
+
+ for (int i = 0; i < customResources.length(); i++) {
+ JSONObject customResource = customResources.getJSONObject(i);
+ assertTrue("Resource type does not have name field: " + customResource,
+ customResource.has("name"));
+ assertTrue("Resource type does not have name resourceType field: "
+ + customResource, customResource.has("resourceType"));
+ assertTrue(
+ "Resource type does not have name units field: " + customResource,
+ customResource.has("units"));
+ assertTrue(
+ "Resource type does not have name value field: " + customResource,
+ customResource.has("value"));
+
+ String name = customResource.getString("name");
+ String unit = customResource.getString("units");
+ String resourceType = customResource.getString("resourceType");
+ Long value = customResource.getLong("value");
+
+ if (ResourceInformation.MEMORY_URI.equals(name)
+ || ResourceInformation.VCORES_URI.equals(name)) {
+ continue;
+ }
+
+ assertTrue("Custom resource type " + name + " not found",
+ customResourceTypes.contains(name));
+ assertEquals("k", unit);
+ assertEquals(ResourceTypes.COUNTABLE,
+ ResourceTypes.valueOf(resourceType));
+ assertNotNull("Custom resource value " + value + " is null!", value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
new file mode 100644
index 0000000..63ae7b7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/FairSchedulerXmlVerifications.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.Node;
+import org.w3c.dom.NodeList;
+
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.XmlCustomResourceTypeTestCase.toXml;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlLong;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.getXmlString;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This test helper class is primarily used by
+ * {@link TestRMWebServicesFairSchedulerCustomResourceTypes}.
+ */
+public class FairSchedulerXmlVerifications {
+
+ private static final Set<String> RESOURCE_FIELDS = Sets.newHashSet(
+ "minResources", "amUsedResources", "amMaxResources", "fairResources",
+ "clusterResources", "reservedResources", "maxResources", "usedResources",
+ "steadyFairResources", "demandResources");
+ private final Set<String> customResourceTypes;
+
+ FairSchedulerXmlVerifications(List<String> customResourceTypes) {
+ this.customResourceTypes = Sets.newHashSet(customResourceTypes);
+ }
+
+ public void verify(Element element) {
+ verifyResourcesContainDefaultResourceTypes(element, RESOURCE_FIELDS);
+ verifyResourcesContainCustomResourceTypes(element, RESOURCE_FIELDS);
+ }
+
+ private void verifyResourcesContainDefaultResourceTypes(Element queue,
+ Set<String> resourceCategories) {
+ for (String resourceCategory : resourceCategories) {
+ boolean hasResourceCategory = hasChild(queue, resourceCategory);
+ assertTrue("Queue " + queue + " does not have resource category key: "
+ + resourceCategory, hasResourceCategory);
+ verifyResourceContainsDefaultResourceTypes(
+ (Element) queue.getElementsByTagName(resourceCategory).item(0));
+ }
+ }
+
+ private void verifyResourceContainsDefaultResourceTypes(
+ Element element) {
+ Object memory = opt(element, "memory");
+ Object vCores = opt(element, "vCores");
+
+ assertNotNull("Key 'memory' not found in: " + element, memory);
+ assertNotNull("Key 'vCores' not found in: " + element, vCores);
+ }
+
+ private void verifyResourcesContainCustomResourceTypes(Element queue,
+ Set<String> resourceCategories) {
+ for (String resourceCategory : resourceCategories) {
+ assertTrue("Queue " + queue + " does not have key for resourceCategory: "
+ + resourceCategory, hasChild(queue, resourceCategory));
+ verifyResourceContainsCustomResourceTypes(
+ (Element) queue.getElementsByTagName(resourceCategory).item(0));
+ }
+ }
+
+ private void verifyResourceContainsCustomResourceTypes(
+ Element resourceCategory) {
+ assertEquals(
+ toXml(resourceCategory)
+ + " should have only one resourceInformations child!",
+ 1, resourceCategory.getElementsByTagName("resourceInformations")
+ .getLength());
+ Element resourceInformations = (Element) resourceCategory
+ .getElementsByTagName("resourceInformations").item(0);
+
+ NodeList customResources =
+ resourceInformations.getElementsByTagName("resourceInformation");
+
+ // customResources will include vcores / memory as well
+ assertEquals(
+ "Different number of custom resource types found than expected",
+ customResourceTypes.size(), customResources.getLength() - 2);
+
+ for (int i = 0; i < customResources.getLength(); i++) {
+ Element customResource = (Element) customResources.item(i);
+ String name = getXmlString(customResource, "name");
+ String unit = getXmlString(customResource, "units");
+ String resourceType = getXmlString(customResource, "resourceType");
+ Long value = getXmlLong(customResource, "value");
+
+ if (ResourceInformation.MEMORY_URI.equals(name)
+ || ResourceInformation.VCORES_URI.equals(name)) {
+ continue;
+ }
+
+ assertTrue("Custom resource type " + name + " not found",
+ customResourceTypes.contains(name));
+ assertEquals("k", unit);
+ assertEquals(ResourceTypes.COUNTABLE,
+ ResourceTypes.valueOf(resourceType));
+ assertNotNull("Resource value should not be null for resource type "
+ + resourceType + ", listing xml contents: " + toXml(customResource),
+ value);
+ }
+ }
+
+ private Object opt(Node node, String child) {
+ NodeList nodes = getElementsByTagNameInternal(node, child);
+ if (nodes.getLength() > 0) {
+ return nodes.item(0);
+ }
+
+ return null;
+ }
+
+ private boolean hasChild(Node node, String child) {
+ return getElementsByTagNameInternal(node, child).getLength() > 0;
+ }
+
+ private NodeList getElementsByTagNameInternal(Node node, String child) {
+ if (node instanceof Element) {
+ return ((Element) node).getElementsByTagName(child);
+ } else if (node instanceof Document) {
+ return ((Document) node).getElementsByTagName(child);
+ } else {
+ throw new IllegalStateException("Unknown type of wrappedObject: " + node
+ + ", type: " + node.getClass());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
new file mode 100644
index 0000000..de4d5a1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/fairscheduler/TestRMWebServicesFairSchedulerCustomResourceTypes.java
@@ -0,0 +1,271 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.fairscheduler;
+
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.JAXBContextResolver;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebServices;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.helper.*;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.w3c.dom.Element;
+import javax.ws.rs.core.MediaType;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * This class is to test response representations of queue resources,
+ * explicitly setting custom resource types. with the help of
+ * {@link CustomResourceTypesConfigurationProvider}
+ */
+public class TestRMWebServicesFairSchedulerCustomResourceTypes
+ extends JerseyTestBase {
+ private static MockRM rm;
+ private static YarnConfiguration conf;
+
+ private static class WebServletModule extends ServletModule {
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+ initResourceTypes(conf);
+ rm = new MockRM(conf);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+
+ private void initResourceTypes(YarnConfiguration conf) {
+ conf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ CustomResourceTypesConfigurationProvider.class.getName());
+ ResourceUtils.resetResourceTypes(conf);
+ }
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ createInjectorForWebServletModule();
+ }
+
+ @After
+ public void tearDown() {
+ ResourceUtils.resetResourceTypes(new Configuration());
+ }
+
+ private void createInjectorForWebServletModule() {
+ GuiceServletConfig
+ .setInjector(Guice.createInjector(new WebServletModule()));
+ }
+
+ @After
+ public void teardown() {
+ CustomResourceTypesConfigurationProvider.reset();
+ }
+
+ public TestRMWebServicesFairSchedulerCustomResourceTypes() {
+ super(new WebAppDescriptor.Builder(
+ "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build());
+ }
+
+ @Test
+ public void testClusterSchedulerWithCustomResourceTypesJson() {
+ FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+ QueueManager queueManager = scheduler.getQueueManager();
+ // create LeafQueues
+ queueManager.getLeafQueue("root.q.subqueue1", true);
+ queueManager.getLeafQueue("root.q.subqueue2", true);
+
+ FSLeafQueue subqueue1 =
+ queueManager.getLeafQueue("root.q.subqueue1", false);
+ incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+ WebResource path =
+ resource().path("ws").path("v1").path("cluster").path("scheduler");
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ verifyJsonResponse(path, response,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+
+ @Test
+ public void testClusterSchedulerWithCustomResourceTypesXml() {
+ FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+ QueueManager queueManager = scheduler.getQueueManager();
+ // create LeafQueues
+ queueManager.getLeafQueue("root.q.subqueue1", true);
+ queueManager.getLeafQueue("root.q.subqueue2", true);
+
+ FSLeafQueue subqueue1 =
+ queueManager.getLeafQueue("root.q.subqueue1", false);
+ incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+ WebResource path =
+ resource().path("ws").path("v1").path("cluster").path("scheduler");
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ verifyXmlResponse(path, response,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+
+ @Test
+ public void testClusterSchedulerWithElevenCustomResourceTypesXml() {
+ CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
+ createInjectorForWebServletModule();
+
+ FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+ QueueManager queueManager = scheduler.getQueueManager();
+ // create LeafQueues
+ queueManager.getLeafQueue("root.q.subqueue1", true);
+ queueManager.getLeafQueue("root.q.subqueue2", true);
+
+ FSLeafQueue subqueue1 =
+ queueManager.getLeafQueue("root.q.subqueue1", false);
+ incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+ WebResource path =
+ resource().path("ws").path("v1").path("cluster").path("scheduler");
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_XML).get(ClientResponse.class);
+
+ verifyXmlResponse(path, response,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+
+ @Test
+ public void testClusterSchedulerElevenWithCustomResourceTypesJson() {
+ CustomResourceTypesConfigurationProvider.setNumberOfResourceTypes(11);
+ createInjectorForWebServletModule();
+
+ FairScheduler scheduler = (FairScheduler) rm.getResourceScheduler();
+ QueueManager queueManager = scheduler.getQueueManager();
+ // create LeafQueues
+ queueManager.getLeafQueue("root.q.subqueue1", true);
+ queueManager.getLeafQueue("root.q.subqueue2", true);
+
+ FSLeafQueue subqueue1 =
+ queueManager.getLeafQueue("root.q.subqueue1", false);
+ incrementUsedResourcesOnQueue(subqueue1, 33L);
+
+ WebResource path =
+ resource().path("ws").path("v1").path("cluster").path("scheduler");
+ ClientResponse response =
+ path.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+
+ verifyJsonResponse(path, response,
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes());
+ }
+
+ private void verifyJsonResponse(WebResource path, ClientResponse response,
+ List<String> customResourceTypes) {
+ JsonCustomResourceTypeTestcase testCase =
+ new JsonCustomResourceTypeTestcase(path,
+ new BufferedClientResponse(response));
+ testCase.verify(json -> {
+ try {
+ JSONArray queues = json.getJSONObject("scheduler")
+ .getJSONObject("schedulerInfo").getJSONObject("rootQueue")
+ .getJSONObject("childQueues").getJSONArray("queue");
+
+ // childQueueInfo consists of subqueue1 and subqueue2 info
+ assertEquals(2, queues.length());
+ JSONObject firstChildQueue = queues.getJSONObject(0);
+ new FairSchedulerJsonVerifications(customResourceTypes)
+ .verify(firstChildQueue);
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ private void verifyXmlResponse(WebResource path, ClientResponse response,
+ List<String> customResourceTypes) {
+ XmlCustomResourceTypeTestCase testCase = new XmlCustomResourceTypeTestCase(
+ path, new BufferedClientResponse(response));
+
+ testCase.verify(xml -> {
+ Element scheduler =
+ (Element) xml.getElementsByTagName("scheduler").item(0);
+ Element schedulerInfo =
+ (Element) scheduler.getElementsByTagName("schedulerInfo").item(0);
+ Element rootQueue =
+ (Element) schedulerInfo.getElementsByTagName("rootQueue").item(0);
+
+ Element childQueues =
+ (Element) rootQueue.getElementsByTagName("childQueues").item(0);
+ Element queue =
+ (Element) childQueues.getElementsByTagName("queue").item(0);
+ new FairSchedulerXmlVerifications(customResourceTypes).verify(queue);
+ });
+ }
+
+ private void incrementUsedResourcesOnQueue(final FSLeafQueue queue,
+ final long value) {
+ try {
+ Method incUsedResourceMethod = queue.getClass().getSuperclass()
+ .getDeclaredMethod("incUsedResource", Resource.class);
+ incUsedResourceMethod.setAccessible(true);
+
+ Map<String, Long> customResources =
+ CustomResourceTypesConfigurationProvider.getCustomResourceTypes()
+ .stream()
+ .collect(Collectors.toMap(Function.identity(), v -> value));
+
+ incUsedResourceMethod.invoke(queue,
+ Resource.newInstance(20, 30, customResources));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99febe7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
new file mode 100644
index 0000000..4ab1443
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/helper/AppInfoJsonVerifications.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.helper;
+
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringEqual;
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.checkStringMatch;
+import static org.junit.Assert.*;
+
+/**
+ * Contains all value verifications that are needed to verify {@link AppInfo}
+ * JSON objects.
+ */
+public final class AppInfoJsonVerifications {
+
+ private AppInfoJsonVerifications() {
+ //utility class
+ }
+
+ /**
+ * Tests whether {@link AppInfo} representation object contains the required
+ * values as per defined in the specified app parameter.
+ * @param app an RMApp instance that contains the required values
+ * to test against.
+ */
+ public static void verify(JSONObject info, RMApp app) throws JSONException {
+ checkStringMatch("id", app.getApplicationId().toString(),
+ info.getString("id"));
+ checkStringMatch("user", app.getUser(), info.getString("user"));
+ checkStringMatch("name", app.getName(), info.getString("name"));
+ checkStringMatch("applicationType", app.getApplicationType(),
+ info.getString("applicationType"));
+ checkStringMatch("queue", app.getQueue(), info.getString("queue"));
+ assertEquals("priority doesn't match", 0, info.getInt("priority"));
+ checkStringMatch("state", app.getState().toString(),
+ info.getString("state"));
+ checkStringMatch("finalStatus", app.getFinalApplicationStatus().toString(),
+ info.getString("finalStatus"));
+ assertEquals("progress doesn't match", 0,
+ (float) info.getDouble("progress"), 0.0);
+ if ("UNASSIGNED".equals(info.getString("trackingUI"))) {
+ checkStringMatch("trackingUI", "UNASSIGNED",
+ info.getString("trackingUI"));
+ }
+ checkStringEqual("diagnostics", app.getDiagnostics().toString(),
+ info.getString("diagnostics"));
+ assertEquals("clusterId doesn't match",
+ ResourceManager.getClusterTimeStamp(), info.getLong("clusterId"));
+ assertEquals("startedTime doesn't match", app.getStartTime(),
+ info.getLong("startedTime"));
+ assertEquals("finishedTime doesn't match", app.getFinishTime(),
+ info.getLong("finishedTime"));
+ assertTrue("elapsed time not greater than 0",
+ info.getLong("elapsedTime") > 0);
+ checkStringMatch("amHostHttpAddress",
+ app.getCurrentAppAttempt().getMasterContainer().getNodeHttpAddress(),
+ info.getString("amHostHttpAddress"));
+ assertTrue("amContainerLogs doesn't match",
+ info.getString("amContainerLogs").startsWith("http://"));
+ assertTrue("amContainerLogs doesn't contain user info",
+ info.getString("amContainerLogs").endsWith("/" + app.getUser()));
+ assertEquals("allocatedMB doesn't match", 1024, info.getInt("allocatedMB"));
+ assertEquals("allocatedVCores doesn't match", 1,
+ info.getInt("allocatedVCores"));
+ assertEquals("queueUsagePerc doesn't match", 50.0f,
+ (float) info.getDouble("queueUsagePercentage"), 0.01f);
+ assertEquals("clusterUsagePerc doesn't match", 50.0f,
+ (float) info.getDouble("clusterUsagePercentage"), 0.01f);
+ assertEquals("numContainers doesn't match", 1,
+ info.getInt("runningContainers"));
+ assertNotNull("preemptedResourceSecondsMap should not be null",
+ info.getJSONObject("preemptedResourceSecondsMap"));
+ assertEquals("preemptedResourceMB doesn't match",
+ app.getRMAppMetrics().getResourcePreempted().getMemorySize(),
+ info.getInt("preemptedResourceMB"));
+ assertEquals("preemptedResourceVCores doesn't match",
+ app.getRMAppMetrics().getResourcePreempted().getVirtualCores(),
+ info.getInt("preemptedResourceVCores"));
+ assertEquals("numNonAMContainerPreempted doesn't match",
+ app.getRMAppMetrics().getNumNonAMContainersPreempted(),
+ info.getInt("numNonAMContainerPreempted"));
+ assertEquals("numAMContainerPreempted doesn't match",
+ app.getRMAppMetrics().getNumAMContainersPreempted(),
+ info.getInt("numAMContainerPreempted"));
+ assertEquals("Log aggregation Status doesn't match",
+ app.getLogAggregationStatusForAppReport().toString(),
+ info.getString("logAggregationStatus"));
+ assertEquals("unmanagedApplication doesn't match",
+ app.getApplicationSubmissionContext().getUnmanagedAM(),
+ info.getBoolean("unmanagedApplication"));
+
+ if (app.getApplicationSubmissionContext()
+ .getNodeLabelExpression() != null) {
+ assertEquals("appNodeLabelExpression doesn't match",
+ app.getApplicationSubmissionContext().getNodeLabelExpression(),
+ info.getString("appNodeLabelExpression"));
+ }
+ assertEquals("amNodeLabelExpression doesn't match",
+ app.getAMResourceRequests().get(0).getNodeLabelExpression(),
+ info.getString("amNodeLabelExpression"));
+ assertEquals("amRPCAddress",
+ AppInfo.getAmRPCAddressFromRMAppAttempt(app.getCurrentAppAttempt()),
+ info.getString("amRPCAddress"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org