You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2019/11/18 09:38:58 UTC
[hadoop] branch branch-3.1 updated: YARN-9900. Revert Invalid
Config and Refresh Support in SchedulerConfig Format.
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d271d17 YARN-9900. Revert Invalid Config and Refresh Support in SchedulerConfig Format.
d271d17 is described below
commit d271d174ca501e43b205b753af4ccfa74e397670
Author: Prabhu Joseph <pj...@cloudera.com>
AuthorDate: Mon Oct 14 17:52:47 2019 +0530
YARN-9900. Revert Invalid Config and Refresh Support in SchedulerConfig Format.
Signed-off-by: prabhujoseph <pj...@cloudera.com>
---
.../scheduler/MutableConfigurationProvider.java | 2 ++
.../conf/MutableCSConfigurationProvider.java | 16 ++++++++++
.../resourcemanager/webapp/RMWebServices.java | 7 +++++
.../TestRMWebServicesConfigurationMutation.java | 35 ++++++++++++++++++++--
4 files changed, 57 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index 9e843df..48c8bb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -67,6 +67,8 @@ public interface MutableConfigurationProvider {
void formatConfigurationInStore(Configuration conf) throws Exception;
+ void revertToOldConfig(Configuration config) throws Exception;
+
/**
* Closes the configuration provider, releasing any required resources.
* @throws IOException on failure to close
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 5e8c915..670d2b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -158,6 +158,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
formatLock.writeLock().lock();
try {
confStore.format();
+ oldConf = new Configuration(schedConf);
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
@@ -177,6 +178,21 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
+ public void revertToOldConfig(Configuration config) throws Exception {
+ formatLock.writeLock().lock();
+ try {
+ schedConf = oldConf;
+ confStore.format();
+ confStore.initialize(config, oldConf, rmContext);
+ confStore.checkVersion();
+ } catch (Exception e) {
+ throw new IOException(e);
+ } finally {
+ formatLock.writeLock().unlock();
+ }
+ }
+
+ @Override
public void confirmPendingMutation(boolean isValid) throws Exception {
formatLock.readLock().lock();
try {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 933a09d..884bcac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2388,6 +2388,13 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
MutableConfigurationProvider mutableConfigurationProvider =
((MutableConfScheduler) scheduler).getMutableConfProvider();
mutableConfigurationProvider.formatConfigurationInStore(conf);
+ try {
+ rm.getRMContext().getRMAdminService().refreshQueues();
+ } catch (IOException | YarnException e) {
+ LOG.error("Exception thrown when formatting configuration.", e);
+ mutableConfigurationProvider.revertToOldConfig(conf);
+ throw e;
+ }
return Response.status(Status.OK).entity("Configuration under " +
"store successfully formatted.").build();
} catch (Exception e) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
index 0584ce8..9ae9dd1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesConfigurationMutation.java
@@ -187,15 +187,44 @@ public class TestRMWebServicesConfigurationMutation extends JerseyTestBase {
@Test
public void testFormatSchedulerConf() throws Exception {
- testAddNestedQueue();
+ CapacitySchedulerConfiguration newConf = getSchedulerConf();
+ assertNotNull(newConf);
+ assertEquals(3, newConf.getQueues("root").length);
+
+ SchedConfUpdateInfo updateInfo = new SchedConfUpdateInfo();
+ Map<String, String> nearEmptyCapacity = new HashMap<>();
+ nearEmptyCapacity.put(CapacitySchedulerConfiguration.CAPACITY, "1E-4");
+ QueueConfigInfo d = new QueueConfigInfo("root.formattest",
+ nearEmptyCapacity);
+ updateInfo.getAddQueueInfo().add(d);
+
+ Map<String, String> stoppedParam = new HashMap<>();
+ stoppedParam.put(CapacitySchedulerConfiguration.STATE,
+ QueueState.STOPPED.toString());
+ QueueConfigInfo stoppedInfo = new QueueConfigInfo("root.formattest",
+ stoppedParam);
+ updateInfo.getUpdateQueueInfo().add(stoppedInfo);
+
+ // Add a queue root.formattest to the existing three queues
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("scheduler-conf").queryParam("user.name", userName)
+ .accept(MediaType.APPLICATION_JSON)
+ .entity(YarnWebServiceUtils.toJson(updateInfo,
+ SchedConfUpdateInfo.class), MediaType.APPLICATION_JSON)
+ .put(ClientResponse.class);
+ newConf = getSchedulerConf();
+ assertNotNull(newConf);
+ assertEquals(4, newConf.getQueues("root").length);
+
+ // Format the scheduler config and validate root.formattest is not present
+ response = r.path("ws").path("v1").path("cluster")
.queryParam("user.name", userName)
.path(RMWSConsts.FORMAT_SCHEDULER_CONF)
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(Status.OK.getStatusCode(), response.getStatus());
- CapacitySchedulerConfiguration orgConf = getSchedulerConf();
- assertEquals(3, orgConf.getQueues("root").length);
+ newConf = getSchedulerConf();
+ assertEquals(3, newConf.getQueues("root").length);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org