You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2020/02/19 05:49:03 UTC
[hadoop] branch branch-3.1 updated: YARN-10139.
ValidateAndGetSchedulerConfiguration API fails when cluster max allocation
> default 8GB. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a6124cd YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
a6124cd is described below
commit a6124cd2b8955306e6acc7a0249871f17e0565e0
Author: Sunil G <su...@apache.org>
AuthorDate: Wed Feb 19 11:17:22 2020 +0530
YARN-10139. ValidateAndGetSchedulerConfiguration API fails when cluster max allocation > default 8GB. Contributed by Prabhu Joseph.
(cherry picked from commit 6526f95bd281fc011f8776d21ff933087c5924de)
---
.../resourcemanager/webapp/RMWebServices.java | 13 +++++++++--
.../TestRMWebServicesConfigurationMutation.java | 26 ++++++++++++++++++++++
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 09b8ea1..e2c8cbb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -31,6 +31,7 @@ import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -2353,14 +2354,22 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
((MutableConfScheduler) scheduler).getMutableConfProvider();
Configuration schedulerConf = mutableConfigurationProvider
.getConfiguration();
- Configuration newConfig = mutableConfigurationProvider
+ Configuration newSchedulerConf = mutableConfigurationProvider
.applyChanges(schedulerConf, mutationInfo);
Configuration yarnConf = ((CapacityScheduler) scheduler).getConf();
+
+ Configuration newConfig = new Configuration(yarnConf);
+ Iterator<Map.Entry<String, String>> iter = newSchedulerConf.iterator();
+ Entry<String, String> e = null;
+ while (iter.hasNext()) {
+ e = iter.next();
+ newConfig.set(e.getKey(), e.getValue());
+ }
CapacitySchedulerConfigValidator.validateCSConfiguration(yarnConf,
newConfig, rm.getRMContext());
return Response.status(Status.OK)
- .entity(new ConfInfo(newConfig))
+ .entity(new ConfInfo(newSchedulerConf))
.build();
} catch (Exception e) {
String errorMsg = "CapacityScheduler configuration validation failed:"
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index accebb9..128559d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -693,6 +693,32 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
newCSConf.getMaximumSystemApplications());
}
+ @Test
+ public void testValidateWithClusterMaxAllocation() throws Exception {
+ WebResource r = resource();
+ int clusterMax = YarnConfiguration.
+ DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB * 2;
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ clusterMax);
+
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ Map<String, String> updateParam = new HashMap<>();
+ updateParam.put(CapacitySchedulerConfiguration.MAXIMUM_APPLICATIONS_SUFFIX,
+ "100");
+ QueueConfigInfo aUpdateInfo = new QueueConfigInfo("root.a", updateParam);
+ updateInfo.getUpdateQueueInfo().add(aUpdateInfo);
+
+ ClientResponse response =
+ r.path("ws").path("v1").path("cluster")
+ .path(RMWSConsts.SCHEDULER_CONF_VALIDATE)
+ .queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class);
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ }
+
@Override
@After
public void tearDown() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org