You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by "tomicooler (via GitHub)" <gi...@apache.org> on 2023/06/19 11:37:46 UTC
[GitHub] [hadoop] tomicooler commented on a diff in pull request #5745: Yarn 11511 - Improve TestRMWebServices test config and data
tomicooler commented on code in PR #5745:
URL: https://github.com/apache/hadoop/pull/5745#discussion_r1233874154
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java:
##########
@@ -297,134 +127,14 @@ public void testResourceInfo() {
assertEquals("<memory:10, vCores:1>", res.toString());
}
- public static void assertXmlType(ClientResponse response) {
- assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- }
-
- public static void assertXmlResponse(ClientResponse response,
- String expectedResourceFilename) throws
- Exception {
- assertXmlType(response);
- Document document = loadDocument(response.getEntity(String.class));
- String actual = serializeDocument(document).trim();
- updateTestDataAutomatically(expectedResourceFilename, actual);
- assertEquals(getResourceAsString(expectedResourceFilename), actual);
- }
-
- public static String serializeDocument(Document document) throws TransformerException {
- DOMSource domSource = new DOMSource(document);
- StringWriter writer = new StringWriter();
- StreamResult result = new StreamResult(writer);
- TransformerFactory tf = XMLUtils.newSecureTransformerFactory();
- Transformer transformer = tf.newTransformer();
- transformer.setOutputProperty(OutputKeys.INDENT, "yes");
- transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
- transformer.transform(domSource, result);
- return writer.toString();
- }
-
- public static Document loadDocument(String xml) throws Exception {
- DocumentBuilderFactory factory = XMLUtils.newSecureDocumentBuilderFactory();
- DocumentBuilder builder = factory.newDocumentBuilder();
- InputSource is = new InputSource(new StringReader(xml));
- return builder.parse(is);
- }
-
- public static void assertJsonResponse(ClientResponse response,
- String expectedResourceFilename) throws
- JSONException, IOException {
- assertJsonType(response);
- JSONObject json = response.getEntity(JSONObject.class);
- String actual = prettyPrintJson(json.toString(2));
- updateTestDataAutomatically(expectedResourceFilename, actual);
- assertEquals(
- prettyPrintJson(getResourceAsString(expectedResourceFilename)),
- prettyPrintJson(actual));
- }
-
- private static String prettyPrintJson(String in) throws JsonProcessingException {
- ObjectMapper objectMapper = new ObjectMapper();
- return objectMapper
- .writerWithDefaultPrettyPrinter()
- .writeValueAsString(objectMapper.readTree(in));
- }
-
- public static void assertJsonType(ClientResponse response) {
- assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- }
-
- public static InputStream getResourceAsStream(String configFilename) {
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- return classLoader.getResourceAsStream(configFilename);
- }
-
- public static String getResourceAsString(String configFilename) throws IOException {
- try (InputStream is = getResourceAsStream(configFilename)) {
- if (is == null) {
- return null;
- }
- try (InputStreamReader isr = new InputStreamReader(is);
- BufferedReader reader = new BufferedReader(isr)) {
- return reader.lines().collect(Collectors.joining(System.lineSeparator()));
- }
- }
- }
-
- public static void updateTestDataAutomatically(String configFilename, String actualContent) {
- /*
- Set UPDATE_TESTDATA=1 environment variable for auto update the expected data
- or uncomment this return statement.
-
- It's safe in a way that, this updates the source directory so the test will still fail,
- because the target directory is untouched.
- */
- if (System.getenv("UPDATE_TESTDATA") == null) {
- return;
- }
-
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- try {
- String resource = Objects.requireNonNull(
- Objects.requireNonNull(classLoader.getResource(configFilename)).toURI().getPath())
- .replaceAll("/target/test-classes/", "/src/test/resources/");
- try (FileWriter writer = new FileWriter(resource, false)) {
- writer.write(actualContent);
- }
- } catch (URISyntaxException | IOException e) {
- e.printStackTrace();
- Assert.fail("overwrite should not fail " + e.getMessage());
- }
- }
-
- public static WebAppDescriptor createWebAppDescriptor() {
- return new WebAppDescriptor.Builder(
- TestRMWebServicesCapacitySched.class.getPackage().getName())
- .contextListenerClass(GuiceServletConfig.class)
- .filterClass(com.google.inject.servlet.GuiceFilter.class)
- .contextPath("jersey-guice-filter").servletPath("/").build();
- }
-
- public static MockRM createMockRM(CapacitySchedulerConfiguration csConf) {
- setupQueueConfiguration(csConf);
- YarnConfiguration conf = new YarnConfiguration(csConf);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
- conf.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
- YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
- return new MockRM(conf);
- }
-
- @Test
- public void testClusterSchedulerOverviewCapacity() throws Exception {
- WebResource r = resource();
- ClientResponse response = r.path("ws").path("v1").path("cluster")
- .path("scheduler-overview").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
- response.getType().toString());
- JSONObject json = response.getEntity(JSONObject.class);
- TestRMWebServices.verifyClusterSchedulerOverView(json, "Capacity Scheduler");
+ private Configuration createConfig() {
+ Configuration conf = new Configuration();
+ conf.set("yarn.scheduler.capacity.root.queues", "a, b, c");
+ conf.set("yarn.scheduler.capacity.root.a.capacity", "20");
+ conf.set("yarn.scheduler.capacity.root.a.maximum-capacity", "50");
+ conf.set("yarn.scheduler.capacity.root.a.max-parallel-app", "42");
+ conf.set("yarn.scheduler.capacity.root.b.capacity", "79.5");
+ conf.set("yarn.scheduler.capacity.root.c.capacity", "0.5");
Review Comment:
This results in a very strange effectiveMin resource when we add some node in the PerUser test.
```
"effectiveMinResource": {
"memory": 51,
"vCores": 0,
```
For 10GB 10VCore we should have some meaningful resource share.
e.g.:
```
a.capacity: 20 (2GB 2 vcore)
b.capacity: 70 (7GB 2 vcore)
c.capacity: 10 (1GB 1 vcore)
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
Review Comment:
It's commented already for the class, can be removed.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java:
##########
@@ -88,8 +90,7 @@
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyNumberOfNodes;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyQueueOrder;
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.ActivitiesTestUtils.verifyStateOfAllocations;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.*;
Review Comment:
Explicit imports.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
Review Comment:
Unused can be removed.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
Review Comment:
rework this test class with reasonable queue hierarchy and cluster resource
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+ /*
+ * mode: percentage
+ * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+ * weight: -1, normalizedWeight: 0
+ * root.queueType: parent, others.queueType: leaf
+ */
+ assertJsonResponse(sendRequest(),
+ "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+ }
+
+ @Test
+ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createAbsoluteConfigLegacyAutoCreation();
Review Comment:
inline
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
Review Comment:
inline the config here
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
Review Comment:
Unused, can be removed.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+ /*
+ * mode: percentage
+ * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+ * weight: -1, normalizedWeight: 0
+ * root.queueType: parent, others.queueType: leaf
+ */
+ assertJsonResponse(sendRequest(),
+ "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+ }
+
+ @Test
+ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createAbsoluteConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+
+ initAutoQueueHandler(8192);
+ createQueue("root.managed.queue1");
Review Comment:
inline
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
@Test
- public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfigLegacyAutoCreation();
-
- initResourceManager(config);
- initAutoQueueHandler(8192 * GB);
- createQueue("root.managed.queue1");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ public void testPercentageMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "37.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testPercentageMode", rm, resource());
}
-
@Test
- public void testSchedulerResponseAbsoluteMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfig();
-
- initResourceManager(config);
-
- /*
- * mode: absolute
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-AbsoluteMode.json");
+ public void testAbsoluteMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=4096,vcores=4]");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "[memory=16384,vcores=16]");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "[memory=12288,vcores=12]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "[memory=12288,vcores=12]");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testAbsoluteMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfig();
-
- initResourceManager(config);
-
- /*
- * mode: weight
- * autoCreationEligibility: off
- * root default test1 test2
- * weight: 1 10 4 6
- * normalizedWeight: 1 0.5 0.2 0.3
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-WeightMode.json");
+ public void testWeightMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testWeightMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfigWithAutoQueueCreationEnabled();
- config.setInt(CapacitySchedulerConfiguration
- .getQueuePrefix("root.autoParent1") +
- AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
- "maximum-applications", 300);
-
- initResourceManager(config);
- initAutoQueueHandler(1200 * GB);
-
- // same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json");
-
- //Now create some auto created queues
- createQueue("root.auto1");
- createQueue("root.auto2");
- createQueue("root.auto3");
- createQueue("root.autoParent1.auto4");
- createQueue("root.autoParent2.auto5");
- createQueue("root.parent.autoParent3.auto6");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json");
- }
-
- private void initAutoQueueHandler(int nodeMemory) throws Exception {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- autoQueueHandler = cs.getCapacitySchedulerQueueManager();
- rm.registerNode("h1:1234", nodeMemory); // label = x
+ public void testWeightModeFlexibleAQC() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+
+ Configuration config = createConfiguration(conf);
+ setupAQC(config, "yarn.scheduler.capacity.root.test1.");
+ setupAQC(config, "yarn.scheduler.capacity.root.test1.test1_1.");
+ try (MockRM rm = createMutableRM(config)) {
+ createAQC(rm, "test1");
Review Comment:
first register the node
then assertjson with a before-aqc suffix
then create the queues
then assertjson with an after-aqc suffix
So we can compare the two jsons.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
@Test
- public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfigLegacyAutoCreation();
-
- initResourceManager(config);
- initAutoQueueHandler(8192 * GB);
- createQueue("root.managed.queue1");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ public void testPercentageMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "37.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testPercentageMode", rm, resource());
}
-
@Test
- public void testSchedulerResponseAbsoluteMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfig();
-
- initResourceManager(config);
-
- /*
- * mode: absolute
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-AbsoluteMode.json");
+ public void testAbsoluteMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=4096,vcores=4]");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "[memory=16384,vcores=16]");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "[memory=12288,vcores=12]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "[memory=12288,vcores=12]");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testAbsoluteMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfig();
-
- initResourceManager(config);
-
- /*
- * mode: weight
- * autoCreationEligibility: off
- * root default test1 test2
- * weight: 1 10 4 6
- * normalizedWeight: 1 0.5 0.2 0.3
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-WeightMode.json");
+ public void testWeightMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+ MockRM rm = createMutableRM(createConfiguration(conf));
Review Comment:
try with resources
```
try (MockRM rm = createMutableRM(createConfiguration(conf))) {
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
@Test
- public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfigLegacyAutoCreation();
-
- initResourceManager(config);
- initAutoQueueHandler(8192 * GB);
- createQueue("root.managed.queue1");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ public void testPercentageMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "37.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testPercentageMode", rm, resource());
}
-
@Test
- public void testSchedulerResponseAbsoluteMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfig();
-
- initResourceManager(config);
-
- /*
- * mode: absolute
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-AbsoluteMode.json");
+ public void testAbsoluteMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=4096,vcores=4]");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "[memory=16384,vcores=16]");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "[memory=12288,vcores=12]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "[memory=12288,vcores=12]");
+ MockRM rm = createMutableRM(createConfiguration(conf));
Review Comment:
try with resources
```
try (MockRM rm = createMutableRM(createConfiguration(conf))) {
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
@Test
- public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfigLegacyAutoCreation();
-
- initResourceManager(config);
- initAutoQueueHandler(8192 * GB);
- createQueue("root.managed.queue1");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ public void testPercentageMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "37.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
+ MockRM rm = createMutableRM(createConfiguration(conf));
Review Comment:
try with resources
```
try (MockRM rm = createMutableRM(createConfiguration(conf))) {
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
@Test
- public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfigLegacyAutoCreation();
-
- initResourceManager(config);
- initAutoQueueHandler(8192 * GB);
- createQueue("root.managed.queue1");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ public void testPercentageMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "37.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testPercentageMode", rm, resource());
}
-
@Test
- public void testSchedulerResponseAbsoluteMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfig();
-
- initResourceManager(config);
-
- /*
- * mode: absolute
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-AbsoluteMode.json");
+ public void testAbsoluteMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=4096,vcores=4]");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "[memory=16384,vcores=16]");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "[memory=12288,vcores=12]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "[memory=12288,vcores=12]");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testAbsoluteMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfig();
-
- initResourceManager(config);
-
- /*
- * mode: weight
- * autoCreationEligibility: off
- * root default test1 test2
- * weight: 1 10 4 6
- * normalizedWeight: 1 0.5 0.2 0.3
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-WeightMode.json");
+ public void testWeightMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testWeightMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfigWithAutoQueueCreationEnabled();
- config.setInt(CapacitySchedulerConfiguration
- .getQueuePrefix("root.autoParent1") +
- AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
- "maximum-applications", 300);
-
- initResourceManager(config);
- initAutoQueueHandler(1200 * GB);
-
- // same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json");
-
- //Now create some auto created queues
- createQueue("root.auto1");
- createQueue("root.auto2");
- createQueue("root.auto3");
- createQueue("root.autoParent1.auto4");
- createQueue("root.autoParent2.auto5");
- createQueue("root.parent.autoParent3.auto6");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json");
- }
-
- private void initAutoQueueHandler(int nodeMemory) throws Exception {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- autoQueueHandler = cs.getCapacitySchedulerQueueManager();
- rm.registerNode("h1:1234", nodeMemory); // label = x
+ public void testWeightModeFlexibleAQC() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+
+ Configuration config = createConfiguration(conf);
+ setupAQC(config, "yarn.scheduler.capacity.root.test1.");
Review Comment:
Too many queues are created, hard to reason about the expected JSON.
How to make it better:
- enable AQC only on one queue (before this change it was the root queue, we can use root.test1 there is 16GB and 16VCore to share)
- create the queues only on one queue
- the effectiveMin resources should make sense on the newly created queues
- the applied template configs should be easily checked, the acl usernames should be meaningful for this to work
- auto-queue-creation-v2.parent-template.acl_ {parentUser, parentAdmin}
- autoParent1.auto-queue-creation-v2.leaf-template.acl_ {ap1User, ap2Admin}
- *.auto-queue-creation-v2.leaf-template.acl_ {leafUser, leafAdmin}
- parent.*.auto-queue-creation-v2.leaf-template.acl {pLeafUser, pLeafAdmin}
Something like this:
```
root.test_1.auto1 [2gb 2vcore] {leafUser, leafAdmin}
root.test_1.auto2 [2gb 2vcore] {leafUser, leafAdmin}
root.test_1.autoParent1.auto3 [1gb 1 vcore] {parentUser, parentAdmin} {ap1User, ap1Admin}
root.test_1.autoParent1.auto4 [1gb 1vcore] {parentUser, parentAdmin} {ap1User, ap1Admin}
root.test_1.autoParent2.auto5 [1gb 1 vcore] {parentUser, parentAdmin} {leafUser, leafAdmin}
root.test_1.parent.autoParent2.auto6 [2gb 2 vcore] {parentUser, parentAdmin} {pLeafUser, pLeafAdmin}
root.test_1.parent2.auto7 [2gb 2 vcore] {parentUser, parentAdmin} {leafUser, leafAdmin}
root.test_1.parent3.auto8 [2gb 2 vcore] {parentUser, parentAdmin} {leafUser, leafAdmin}
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedDynamicConfig.java:
##########
@@ -18,311 +18,161 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.google.inject.Guice;
-import com.sun.jersey.api.client.ClientResponse;
-
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import javax.ws.rs.core.MediaType;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedQueueTemplate;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.junit.Test;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerTestUtilities.GB;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.assertJsonResponse;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createMockRM;
-import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestRMWebServicesCapacitySched.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.reinitialize;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.runTest;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.sendRequest;
+import static org.assertj.core.api.Assertions.fail;
+
+
+/*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
+public class TestRMWebServicesCapacitySchedDynamicConfig extends JerseyTestBase {
-public class TestRMWebServicesCapacitySchedDynamicConfig extends
- JerseyTestBase {
+ private static final String EXPECTED_FILE_TMPL = "webapp/dynamic-%s-%s.json";
private MockRM rm;
- private CapacitySchedulerQueueManager autoQueueHandler;
-
public TestRMWebServicesCapacitySchedDynamicConfig() {
super(createWebAppDescriptor());
}
- @Test
- public void testSchedulerResponsePercentageMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfig();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-PercentageMode.json");
- }
-
- @Test
- public void testSchedulerResponsePercentageModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createPercentageConfigLegacyAutoCreation();
-
- initResourceManager(config);
-
- /*
- * mode: percentage
- * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
- }
+ private static final String[] SETUP = new String[] {
+ "yarn.scheduler.capacity.root.queues = default, test1, test2",
+ "yarn.scheduler.capacity.root.test1.queues = test1_1, test1_2, test1_3"
+ };
+
+ /*
+ * EffectiveMin (32GB 32VCores) AbsoluteCapacity
+ * root.default 4/32 [memory=4096, vcores=4] 12.5%
+ * root.test_1 16/32 [memory=16384, vcores=16]
+ * root.test_2 12/32 [memory=12288, vcores=12] 37.5%
+ * root.test_1.test_1_1 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_2 2/16 [memory=2048, vcores=2] 6.25%
+ * root.test_1.test_1_3 12/16 [memory=12288, vcores=12] 37.5%
+ */
@Test
- public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfigLegacyAutoCreation();
-
- initResourceManager(config);
- initAutoQueueHandler(8192 * GB);
- createQueue("root.managed.queue1");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ public void testPercentageMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "50");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "37.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "12.5");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "75");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testPercentageMode", rm, resource());
}
-
@Test
- public void testSchedulerResponseAbsoluteMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createAbsoluteConfig();
-
- initResourceManager(config);
-
- /*
- * mode: absolute
- * autoCreationEligibility: off
- * weight: -1, normalizedWeight: 0
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-AbsoluteMode.json");
+ public void testAbsoluteMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "[memory=4096,vcores=4]");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "[memory=16384,vcores=16]");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "[memory=12288,vcores=12]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "[memory=2048,vcores=2]");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "[memory=12288,vcores=12]");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testAbsoluteMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightMode()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfig();
-
- initResourceManager(config);
-
- /*
- * mode: weight
- * autoCreationEligibility: off
- * root default test1 test2
- * weight: 1 10 4 6
- * normalizedWeight: 1 0.5 0.2 0.3
- * root.queueType: parent, others.queueType: leaf
- */
- assertJsonResponse(sendRequest(), "webapp/scheduler-response-WeightMode.json");
+ public void testWeightMode() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+ MockRM rm = createMutableRM(createConfiguration(conf));
+ runTest(EXPECTED_FILE_TMPL, "testWeightMode", rm, resource());
}
@Test
- public void testSchedulerResponseWeightModeWithAutoCreatedQueues()
- throws Exception {
- Configuration config = CSConfigGenerator
- .createWeightConfigWithAutoQueueCreationEnabled();
- config.setInt(CapacitySchedulerConfiguration
- .getQueuePrefix("root.autoParent1") +
- AutoCreatedQueueTemplate.AUTO_QUEUE_TEMPLATE_PREFIX +
- "maximum-applications", 300);
-
- initResourceManager(config);
- initAutoQueueHandler(1200 * GB);
-
- // same as webapp/scheduler-response-WeightMode.json, but with effective resources filled in
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-Before.json");
-
- //Now create some auto created queues
- createQueue("root.auto1");
- createQueue("root.auto2");
- createQueue("root.auto3");
- createQueue("root.autoParent1.auto4");
- createQueue("root.autoParent2.auto5");
- createQueue("root.parent.autoParent3.auto6");
-
- assertJsonResponse(sendRequest(),
- "webapp/scheduler-response-WeightModeWithAutoCreatedQueues-After.json");
- }
-
- private void initAutoQueueHandler(int nodeMemory) throws Exception {
- CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- autoQueueHandler = cs.getCapacitySchedulerQueueManager();
- rm.registerNode("h1:1234", nodeMemory); // label = x
+ public void testWeightModeFlexibleAQC() throws Exception {
+ Map<String, String> conf = new HashMap<>();
+ conf.put("yarn.scheduler.capacity.root.queues", "default, test1, test2");
+ conf.put("yarn.scheduler.capacity.root.test1.queues", "test1_1, test1_2, test1_3");
+ conf.put("yarn.scheduler.capacity.root.default.capacity", "4w");
+ conf.put("yarn.scheduler.capacity.root.test1.capacity", "16w");
+ conf.put("yarn.scheduler.capacity.root.test2.capacity", "12w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_1.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_2.capacity", "2w");
+ conf.put("yarn.scheduler.capacity.root.test1.test1_3.capacity", "12w");
+
+ Configuration config = createConfiguration(conf);
+ setupAQC(config, "yarn.scheduler.capacity.root.test1.");
+ setupAQC(config, "yarn.scheduler.capacity.root.test1.test1_1.");
+ try (MockRM rm = createMutableRM(config)) {
+ createAQC(rm, "test1");
+ createAQC(rm, "test1.test1_1");
+ rm.registerNode("h1:1234", 32768, 32);
Review Comment:
32 * GB
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java:
##########
@@ -18,273 +18,103 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
-
-import java.io.BufferedReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.StringReader;
-import java.io.StringWriter;
-import java.net.URISyntaxException;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
import javax.ws.rs.core.MediaType;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-import javax.xml.transform.OutputKeys;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.dom.DOMSource;
-import javax.xml.transform.stream.StreamResult;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Test;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.JettyUtils;
-import org.apache.hadoop.util.XMLUtils;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmissionData;
import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
-import org.codehaus.jettison.json.JSONException;
-import org.codehaus.jettison.json.JSONObject;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import org.w3c.dom.Document;
-import org.xml.sax.InputSource;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.CAPACITY;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.MAX_PARALLEL_APPLICATIONS;
-import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonType;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertXmlResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
import static org.junit.Assert.assertEquals;
public class TestRMWebServicesCapacitySched extends JerseyTestBase {
- private MockRM rm;
-
- public static class WebServletModule extends ServletModule {
- private final MockRM rm;
-
- WebServletModule(MockRM rm) {
- this.rm = rm;
- }
-
- @Override
- protected void configureServlets() {
- bind(JAXBContextResolver.class);
- bind(RMWebServices.class);
- bind(GenericExceptionHandler.class);
- bind(ResourceManager.class).toInstance(rm);
- serve("/*").with(GuiceContainer.class);
- }
- }
-
public TestRMWebServicesCapacitySched() {
super(createWebAppDescriptor());
}
- @Before
- @Override
- public void setUp() throws Exception {
- super.setUp();
- rm = createMockRM(new CapacitySchedulerConfiguration(
- new Configuration(false)));
- GuiceServletConfig.setInjector(
- Guice.createInjector(new WebServletModule(rm)));
- }
-
- public static void setupQueueConfiguration(
- CapacitySchedulerConfiguration config) {
-
- // Define top-level queues
- config.setQueues(CapacitySchedulerConfiguration.ROOT,
- new String[] {"a", "b", "c"});
-
- final String a = CapacitySchedulerConfiguration.ROOT + ".a";
- config.setCapacity(a, 10.5f);
- config.setMaximumCapacity(a, 50);
- config.setInt(CapacitySchedulerConfiguration.getQueuePrefix(a) + MAX_PARALLEL_APPLICATIONS, 42);
-
- final String b = CapacitySchedulerConfiguration.ROOT + ".b";
- config.setCapacity(b, 89.5f);
-
- final String c = CapacitySchedulerConfiguration.ROOT + ".c";
- config.setCapacity(c, "[memory=1024]");
-
- // Define 2nd-level queues
- final String a1 = a + ".a1";
- final String a2 = a + ".a2";
- config.setQueues(a, new String[] {"a1", "a2"});
- config.setCapacity(a1, 30);
- config.setMaximumCapacity(a1, 50);
- config.setMaximumLifetimePerQueue(a2, 100);
- config.setDefaultLifetimePerQueue(a2, 50);
-
- config.setUserLimitFactor(a1, 100.0f);
- config.setCapacity(a2, 70);
- config.setUserLimitFactor(a2, 100.0f);
-
- final String b1 = b + ".b1";
- final String b2 = b + ".b2";
- final String b3 = b + ".b3";
- config.setQueues(b, new String[] {"b1", "b2", "b3"});
- config.setCapacity(b1, 60);
- config.setUserLimitFactor(b1, 100.0f);
- config.setCapacity(b2, 39.5f);
- config.setUserLimitFactor(b2, 100.0f);
- config.setCapacity(b3, 0.5f);
- config.setUserLimitFactor(b3, 100.0f);
-
- config.setQueues(a1, new String[] {"a1a", "a1b", "a1c"});
- final String a1A = a1 + ".a1a";
- config.setCapacity(a1A, 65);
- final String a1B = a1 + ".a1b";
- config.setCapacity(a1B, 15);
- final String a1C = a1 + ".a1c";
- config.setCapacity(a1C, 20);
-
- config.setAutoCreateChildQueueEnabled(a1C, true);
- config.setInt(PREFIX + a1C + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX
- + DOT + CAPACITY, 50);
- }
-
@Test
public void testClusterScheduler() throws Exception {
- ClientResponse response = resource().path("ws").path("v1").path("cluster")
- .path("scheduler").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- assertJsonResponse(response, "webapp/scheduler-response.json");
- }
-
- @Test
- public void testClusterSchedulerSlash() throws Exception {
- ClientResponse response = resource().path("ws").path("v1").path("cluster")
- .path("scheduler/").accept(MediaType.APPLICATION_JSON)
- .get(ClientResponse.class);
- assertJsonResponse(response, "webapp/scheduler-response.json");
- }
-
- @Test
- public void testClusterSchedulerDefault() throws Exception {
- ClientResponse response = resource().path("ws").path("v1").path("cluster")
- .path("scheduler").get(ClientResponse.class);
- assertJsonResponse(response, "webapp/scheduler-response.json");
- }
-
- @Test
- public void testClusterSchedulerXML() throws Exception {
- ClientResponse response = resource().path("ws").path("v1").path("cluster")
- .path("scheduler/").accept(MediaType.APPLICATION_XML)
- .get(ClientResponse.class);
- assertXmlResponse(response, "webapp/scheduler-response.xml");
+ try (MockRM rm = createRM(createConfig())){
+ assertJsonResponse(resource().path("ws/v1/cluster/scheduler")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class),
+ "webapp/scheduler-response.json");
+ assertJsonResponse(resource().path("ws/v1/cluster/scheduler/")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class),
+ "webapp/scheduler-response.json");
+ assertJsonResponse(resource().path("ws/v1/cluster/scheduler")
+ .get(ClientResponse.class)
+ , "webapp/scheduler-response.json");
+ assertXmlResponse(resource().path("ws/v1/cluster/scheduler/")
+ .accept(MediaType.APPLICATION_XML).get(ClientResponse.class),
+ "webapp/scheduler-response.xml");
+ }
}
@Test
- public void testPerUserResourcesXML() throws Exception {
- // Start RM so that it accepts app submissions
- rm.start();
- try {
- MockRMAppSubmissionData data1 =
- MockRMAppSubmissionData.Builder.createWithMemory(10, rm)
- .withAppName("app1")
- .withUser("user1")
- .withAcls(null)
- .withQueue("b1")
- .withUnmanagedAM(false)
- .build();
- MockRMAppSubmitter.submit(rm, data1);
- MockRMAppSubmissionData data =
- MockRMAppSubmissionData.Builder.createWithMemory(20, rm)
- .withAppName("app2")
- .withUser("user2")
- .withAcls(null)
- .withQueue("b1")
- .withUnmanagedAM(false)
- .build();
- MockRMAppSubmitter.submit(rm, data);
+ public void testPerUserResources() throws Exception {
+ try (MockRM rm = createRM(createConfig())){
+ rm.registerNode("h1:1234", 10240, 10);
Review Comment:
Nit: 10 * GB
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebServiceUtil.java:
##########
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.BufferedReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.net.URISyntaxException;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Assert;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.MEMORY_CONFIGURATION_STORE;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS;
+import static org.junit.Assert.assertEquals;
+
+public final class TestWebServiceUtil {
+ private TestWebServiceUtil(){
+ }
+
+ public static class WebServletModule extends ServletModule {
+ private final MockRM rm;
+
+ WebServletModule(MockRM rm) {
+ this.rm = rm;
+ }
+
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+ }
+
+ public static void runTest(String template, String name,
+ MockRM rm,
+ WebResource resource) throws Exception {
+ final boolean reinitAfterNodeChane = isMutableConfig(rm.getConfig());
+ try {
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 0));
+ MockNM nm1 = rm.registerNode("h1:1234", 8192, 8);
+ rm.registerNode("h2:1234", 8192, 8);
+ if (reinitAfterNodeChane) {
+ reinitialize(rm, rm.getConfig());
+ }
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 16));
+ rm.registerNode("h3:1234", 8192, 8);
+ MockNM nm4 = rm.registerNode("h4:1234", 8192, 8);
+ if (reinitAfterNodeChane) {
+ reinitialize(rm, rm.getConfig());
+ }
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 32));
+ rm.unRegisterNode(nm1);
+ rm.unRegisterNode(nm4);
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 16));
+ } finally {
+ rm.close();
+ }
+ }
+ public static boolean isMutableConfig(Configuration config) {
+ return Objects.equals(config.get(SCHEDULER_CONFIGURATION_STORE_CLASS),
+ MEMORY_CONFIGURATION_STORE);
+ }
+
+ public static ClientResponse sendRequest(WebResource resource) {
+ return resource.path("ws").path("v1").path("cluster")
+ .path("scheduler").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ }
+
+ public static void assertXmlType(ClientResponse response) {
+ assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ }
+
+ public static void assertXmlResponse(ClientResponse response,
+ String expectedResourceFilename) throws
+ Exception {
+ assertXmlType(response);
+ Document document = loadDocument(response.getEntity(String.class));
+ String actual = serializeDocument(document).trim();
+ updateTestDataAutomatically(expectedResourceFilename, actual);
+ assertEquals(getResourceAsString(expectedResourceFilename), actual);
+ }
+
+ public static String serializeDocument(Document document) throws TransformerException {
+ DOMSource domSource = new DOMSource(document);
+ StringWriter writer = new StringWriter();
+ StreamResult result = new StreamResult(writer);
+ TransformerFactory tf = XMLUtils.newSecureTransformerFactory();
+ Transformer transformer = tf.newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+ transformer.transform(domSource, result);
+ return writer.toString();
+ }
+
+ public static Document loadDocument(String xml) throws Exception {
+ DocumentBuilderFactory factory = XMLUtils.newSecureDocumentBuilderFactory();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ InputSource is = new InputSource(new StringReader(xml));
+ return builder.parse(is);
+ }
+
+ public static void assertJsonResponse(ClientResponse response,
+ String expectedResourceFilename) throws JSONException,
+ IOException {
+ assertJsonType(response);
+ JSONObject json = response.getEntity(JSONObject.class);
+ String actual = prettyPrintJson(json.toString(2));
+ updateTestDataAutomatically(expectedResourceFilename, actual);
+ assertEquals(
+ prettyPrintJson(getResourceAsString(expectedResourceFilename)),
+ actual);
+ }
+
+ private static String prettyPrintJson(String in) throws JsonProcessingException {
+ ObjectMapper objectMapper = new ObjectMapper();
+ return objectMapper
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(objectMapper.readTree(in));
+ }
+
+ public static void assertJsonType(ClientResponse response) {
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ }
+
+ public static InputStream getResourceAsStream(String configFilename) {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ return classLoader.getResourceAsStream(configFilename);
+ }
+
+ public static String getResourceAsString(String configFilename) throws IOException {
+ try (InputStream is = getResourceAsStream(configFilename)) {
+ if (is == null) {
+ return null;
+ }
+ try (InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader reader = new BufferedReader(isr)) {
+ return reader.lines().collect(Collectors.joining(System.lineSeparator()));
+ }
+ }
+ }
+
+ public static void updateTestDataAutomatically(String configFilename, String actualContent) {
+ /*
+ Set UPDATE_TESTDATA=1 environment variable for auto update the expected data
+ or uncomment this return statement.
+
+ It's safe in a way that, this updates the source directory so the test will still fail,
+ because the target directory is untouched.
+ */
+ if (System.getenv("UPDATE_TESTDATA") == null) {
+ return;
+ }
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ String resource = Objects.requireNonNull(
+ Objects.requireNonNull(classLoader.getResource(configFilename)).toURI().getPath())
+ .replaceAll("/target/test-classes/", "/src/test/resources/");
+ try (FileWriter writer = new FileWriter(resource, false)) {
+ writer.write(actualContent);
+ }
+ } catch (URISyntaxException | IOException e) {
+ e.printStackTrace();
+ Assert.fail("overwrite should not fail " + e.getMessage());
+ }
+ }
+ public static WebAppDescriptor createWebAppDescriptor() {
+ return new WebAppDescriptor.Builder(
+ TestRMWebServicesCapacitySched.class.getPackage().getName())
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build();
+ }
+
+ public static CapacitySchedulerConfiguration createConfig(CapacitySchedulerConfiguration config) {
+ config.set("yarn.scheduler.capacity.root.queues", "a, b, c");
+ config.set("yarn.scheduler.capacity.root.a.queues", "a1, a2");
+ config.set("yarn.scheduler.capacity.root.b.queues", "b1, b2, b3");
+ config.set("yarn.scheduler.capacity.root.a.a1.queues", "a1a, a1b, a1c");
+ config.set("yarn.scheduler.capacity.root.a.capacity", "10.5");
+ config.set("yarn.scheduler.capacity.root.a.maximum-capacity", "50");
+ config.set("yarn.scheduler.capacity.root.a.max-parallel-app", "42");
+ config.set("yarn.scheduler.capacity.root.b.capacity", "79.5");
+ config.set("yarn.scheduler.capacity.root.c.capacity", "10");
+ config.set("yarn.scheduler.capacity.root.a.a1.capacity", "30");
+ config.set("yarn.scheduler.capacity.root.a.a1.maximum-capacity", "50");
+ config.set("yarn.scheduler.capacity.root.a.a1.user-limit-factor", "100");
+ config.set("yarn.scheduler.capacity.root.a.a2.capacity", "70");
+ config.set("yarn.scheduler.capacity.root.a.a2.maximum-application-lifetime", "100");
+ config.set("yarn.scheduler.capacity.root.a.a2.default-application-lifetime", "50");
+ config.set("yarn.scheduler.capacity.root.a.a2.user-limit-factor", "100");
+ config.set("yarn.scheduler.capacity.root.b.b1.capacity", "60");
+ config.set("yarn.scheduler.capacity.root.b.b2.capacity", "39.5");
+ config.set("yarn.scheduler.capacity.root.b.b3.capacity", "0.5");
+ config.set("yarn.scheduler.capacity.root.b.b1.user-limit-factor", "100");
+ config.set("yarn.scheduler.capacity.root.b.b2.user-limit-factor", "100");
+ config.set("yarn.scheduler.capacity.root.b.b3.user-limit-factor", "100");
+ config.set("yarn.scheduler.capacity.root.a.a1.a1a.capacity", "65");
+ config.set("yarn.scheduler.capacity.root.a.a1.a1b.capacity", "15");
+ config.set("yarn.scheduler.capacity.root.a.a1.a1c.capacity", "20");
+ config.set("yarn.scheduler.capacity.root.a.a1.a1c.auto-create-child-queue.enabled", "true");
+ config.set("yarn.scheduler.capacity.root.a.a1.a1c.leaf-queue-template.capacity", "50");
+ return config;
+ }
+
+ public static MockRM createRM(Configuration config) {
+ config.setClass(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class, ResourceScheduler.class);
+ config.set(YarnConfiguration.RM_PLACEMENT_CONSTRAINTS_HANDLER,
+ YarnConfiguration.SCHEDULER_RM_PLACEMENT_CONSTRAINTS_HANDLER);
+ MockRM rm = new MockRM(config);
+ GuiceServletConfig.setInjector(Guice.createInjector(new WebServletModule(rm)));
+ rm.start();
+ return rm;
+ }
+
+ public static MockRM createMutableRM(Configuration conf) throws IOException {
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+ MockRM rm = createRM(new CapacitySchedulerConfiguration(conf));
+ reinitialize(rm, conf);
+ return rm;
+ }
+
+ public static void reinitialize(MockRM rm, Configuration conf) throws IOException {
+ // Need to call reinitialize as
+ // MutableCSConfigurationProvider with InMemoryConfigurationStore
+ // somehow does not load the queues properly and falls back to default config.
+ // Therefore CS will think there's only the default queue there.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ cs.reinitialize(conf, rm.getRMContext(), true);
+ }
+
+// public static MockRM createMockRM(CapacitySchedulerConfiguration config) {
Review Comment:
remove this
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesSchedulerActivities.java:
##########
@@ -399,7 +442,7 @@ public void testReserveNewContainer() throws Exception {
JSONObject allocations = getFirstSubNodeFromJson(json,
FN_SCHEDULER_ACT_ROOT, FN_ACT_ALLOCATIONS);
verifyQueueOrder(allocations,
- "root-root.c-root.a-root.b-root.b.b3-root.b.b1");
+ "root-root.a-root.c-root.b-root.b.b3-root.b.b1");
Review Comment:
Why did this change?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestWebServiceUtil.java:
##########
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.BufferedReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.StringReader;
+import java.io.StringWriter;
+import java.net.URISyntaxException;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.OutputKeys;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.Assert;
+import org.w3c.dom.Document;
+import org.xml.sax.InputSource;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.JettyUtils;
+import org.apache.hadoop.util.XMLUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.MEMORY_CONFIGURATION_STORE;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS;
+import static org.junit.Assert.assertEquals;
+
+public final class TestWebServiceUtil {
+ private TestWebServiceUtil(){
+ }
+
+ public static class WebServletModule extends ServletModule {
+ private final MockRM rm;
+
+ WebServletModule(MockRM rm) {
+ this.rm = rm;
+ }
+
+ @Override
+ protected void configureServlets() {
+ bind(JAXBContextResolver.class);
+ bind(RMWebServices.class);
+ bind(GenericExceptionHandler.class);
+ bind(ResourceManager.class).toInstance(rm);
+ serve("/*").with(GuiceContainer.class);
+ }
+ }
+
+ public static void runTest(String template, String name,
+ MockRM rm,
+ WebResource resource) throws Exception {
+ final boolean reinitAfterNodeChane = isMutableConfig(rm.getConfig());
+ try {
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 0));
+ MockNM nm1 = rm.registerNode("h1:1234", 8192, 8);
+ rm.registerNode("h2:1234", 8192, 8);
+ if (reinitAfterNodeChane) {
+ reinitialize(rm, rm.getConfig());
+ }
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 16));
+ rm.registerNode("h3:1234", 8192, 8);
+ MockNM nm4 = rm.registerNode("h4:1234", 8192, 8);
+ if (reinitAfterNodeChane) {
+ reinitialize(rm, rm.getConfig());
+ }
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 32));
+ rm.unRegisterNode(nm1);
+ rm.unRegisterNode(nm4);
+ assertJsonResponse(sendRequest(resource), String.format(template, name, 16));
+ } finally {
+ rm.close();
+ }
+ }
+ public static boolean isMutableConfig(Configuration config) {
+ return Objects.equals(config.get(SCHEDULER_CONFIGURATION_STORE_CLASS),
+ MEMORY_CONFIGURATION_STORE);
+ }
+
+ public static ClientResponse sendRequest(WebResource resource) {
+ return resource.path("ws").path("v1").path("cluster")
+ .path("scheduler").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ }
+
+ public static void assertXmlType(ClientResponse response) {
+ assertEquals(MediaType.APPLICATION_XML_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ }
+
+ public static void assertXmlResponse(ClientResponse response,
+ String expectedResourceFilename) throws
+ Exception {
+ assertXmlType(response);
+ Document document = loadDocument(response.getEntity(String.class));
+ String actual = serializeDocument(document).trim();
+ updateTestDataAutomatically(expectedResourceFilename, actual);
+ assertEquals(getResourceAsString(expectedResourceFilename), actual);
+ }
+
+ public static String serializeDocument(Document document) throws TransformerException {
+ DOMSource domSource = new DOMSource(document);
+ StringWriter writer = new StringWriter();
+ StreamResult result = new StreamResult(writer);
+ TransformerFactory tf = XMLUtils.newSecureTransformerFactory();
+ Transformer transformer = tf.newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2");
+ transformer.transform(domSource, result);
+ return writer.toString();
+ }
+
+ public static Document loadDocument(String xml) throws Exception {
+ DocumentBuilderFactory factory = XMLUtils.newSecureDocumentBuilderFactory();
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ InputSource is = new InputSource(new StringReader(xml));
+ return builder.parse(is);
+ }
+
+ public static void assertJsonResponse(ClientResponse response,
+ String expectedResourceFilename) throws JSONException,
+ IOException {
+ assertJsonType(response);
+ JSONObject json = response.getEntity(JSONObject.class);
+ String actual = prettyPrintJson(json.toString(2));
+ updateTestDataAutomatically(expectedResourceFilename, actual);
+ assertEquals(
+ prettyPrintJson(getResourceAsString(expectedResourceFilename)),
+ actual);
+ }
+
+ private static String prettyPrintJson(String in) throws JsonProcessingException {
+ ObjectMapper objectMapper = new ObjectMapper();
+ return objectMapper
+ .writerWithDefaultPrettyPrinter()
+ .writeValueAsString(objectMapper.readTree(in));
+ }
+
+ public static void assertJsonType(ClientResponse response) {
+ assertEquals(MediaType.APPLICATION_JSON_TYPE + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ }
+
+ public static InputStream getResourceAsStream(String configFilename) {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ return classLoader.getResourceAsStream(configFilename);
+ }
+
+ public static String getResourceAsString(String configFilename) throws IOException {
+ try (InputStream is = getResourceAsStream(configFilename)) {
+ if (is == null) {
+ return null;
+ }
+ try (InputStreamReader isr = new InputStreamReader(is);
+ BufferedReader reader = new BufferedReader(isr)) {
+ return reader.lines().collect(Collectors.joining(System.lineSeparator()));
+ }
+ }
+ }
+
+ public static void updateTestDataAutomatically(String configFilename, String actualContent) {
+ /*
+ Set UPDATE_TESTDATA=1 environment variable for auto update the expected data
+ or uncomment this return statement.
+
+ It's safe in a way that, this updates the source directory so the test will still fail,
+ because the target directory is untouched.
+ */
+ if (System.getenv("UPDATE_TESTDATA") == null) {
+ return;
+ }
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ try {
+ String resource = Objects.requireNonNull(
+ Objects.requireNonNull(classLoader.getResource(configFilename)).toURI().getPath())
+ .replaceAll("/target/test-classes/", "/src/test/resources/");
+ try (FileWriter writer = new FileWriter(resource, false)) {
+ writer.write(actualContent);
+ }
+ } catch (URISyntaxException | IOException e) {
+ e.printStackTrace();
+ Assert.fail("overwrite should not fail " + e.getMessage());
+ }
+ }
+ public static WebAppDescriptor createWebAppDescriptor() {
+ return new WebAppDescriptor.Builder(
+ TestRMWebServicesCapacitySched.class.getPackage().getName())
+ .contextListenerClass(GuiceServletConfig.class)
+ .filterClass(com.google.inject.servlet.GuiceFilter.class)
+ .contextPath("jersey-guice-filter").servletPath("/").build();
+ }
+
+ public static CapacitySchedulerConfiguration createConfig(CapacitySchedulerConfiguration config) {
Review Comment:
remove this from here
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
Review Comment:
no need for this, see my comment above
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
Review Comment:
try with resources
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
Review Comment:
no need for member variable, create the mutable rm with try resource like in the dynamic file
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+ /*
+ * mode: percentage
+ * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+ * weight: -1, normalizedWeight: 0
+ * root.queueType: parent, others.queueType: leaf
+ */
+ assertJsonResponse(sendRequest(),
+ "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+ }
+
+ @Test
+ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createAbsoluteConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+
+ initAutoQueueHandler(8192);
Review Comment:
inline
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+ /*
+ * mode: percentage
+ * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+ * weight: -1, normalizedWeight: 0
+ * root.queueType: parent, others.queueType: leaf
+ */
+ assertJsonResponse(sendRequest(),
+ "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+ }
+
+ @Test
+ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createAbsoluteConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
Review Comment:
try with resources
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySchedLegacyQueueCreation.java:
##########
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import javax.ws.rs.core.MediaType;
+
+import com.sun.jersey.api.client.ClientResponse;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueuePath;
+import org.apache.hadoop.yarn.webapp.JerseyTestBase;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfigGeneratorForTest.createConfiguration;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.assertJsonResponse;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createMutableRM;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.TestWebServiceUtil.createWebAppDescriptor;
+
+public class TestRMWebServicesCapacitySchedLegacyQueueCreation extends
+ JerseyTestBase {
+ private MockRM rm;
+
+ private CapacitySchedulerQueueManager autoQueueHandler;
+
+ public TestRMWebServicesCapacitySchedLegacyQueueCreation() {
+ super(createWebAppDescriptor());
+ }
+
+ @After
+ public void shutDown(){
+ if (rm != null) {
+ rm.stop();
+ rm = null;
+ }
+ }
+
+ @Test
+ public void testSchedulerResponsePercentageModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createPercentageConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+ /*
+ * mode: percentage
+ * managedtest2.autoCreationEligibility: legacy, others.autoCreationEligibility: off
+ * weight: -1, normalizedWeight: 0
+ * root.queueType: parent, others.queueType: leaf
+ */
+ assertJsonResponse(sendRequest(),
+ "webapp/scheduler-response-PercentageModeLegacyAutoCreation.json");
+ }
+
+ @Test
+ public void testSchedulerResponseAbsoluteModeLegacyAutoCreation()
+ throws Exception {
+ Configuration config = createAbsoluteConfigLegacyAutoCreation();
+
+ rm = createMutableRM(config);
+
+ initAutoQueueHandler(8192);
+ createQueue("root.managed.queue1");
+
+ assertJsonResponse(sendRequest(),
+ "webapp/scheduler-response-AbsoluteModeLegacyAutoCreation.json");
+ }
+
+ private void initAutoQueueHandler(int nodeMemory) throws Exception {
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ autoQueueHandler = cs.getCapacitySchedulerQueueManager();
+ rm.registerNode("h1:1234", nodeMemory, 32); // label = x
Review Comment:
32 * GB, 32
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org