You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by pr...@apache.org on 2019/11/19 10:58:56 UTC
[hadoop] branch branch-3.1 updated: YARN-9909. Offline Format of
YarnConfigurationStore. Contributed by Prabhu Joseph.
This is an automated email from the ASF dual-hosted git repository.
prabhujoseph pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new dab4cc6 YARN-9909. Offline Format of YarnConfigurationStore. Contributed by Prabhu Joseph.
dab4cc6 is described below
commit dab4cc6320b105831cdaa8f8cd4078c613b8a219
Author: Sunil G <su...@apache.org>
AuthorDate: Thu Oct 17 01:00:08 2019 +0530
YARN-9909. Offline Format of YarnConfigurationStore. Contributed by Prabhu Joseph.
---
.../server/resourcemanager/ResourceManager.java | 47 +++++++++++++++++++++-
.../conf/MutableCSConfigurationProvider.java | 18 +--------
.../conf/YarnConfigurationStoreFactory.java | 22 +++++++---
.../resourcemanager/TestRMStoreCommands.java | 35 ++++++++++++++++
.../src/site/markdown/YarnCommands.md | 2 +-
5 files changed, 100 insertions(+), 24 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index e3544ff..e368e26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -96,11 +96,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.AllocationTagsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.MemoryPlacementConstraintManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.constraint.PlacementConstraintManagerService;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.MutableConfScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
@@ -1491,6 +1494,8 @@ public class ResourceManager extends CompositeService
if (argv.length >= 1) {
if (argv[0].equals("-format-state-store")) {
deleteRMStateStore(conf);
+ } else if (argv[0].equals("-format-conf-store")) {
+ deleteRMConfStore(conf);
} else if (argv[0].equals("-remove-application-from-state-store")
&& argv.length == 2) {
removeApplication(conf, argv[1]);
@@ -1583,6 +1588,44 @@ public class ResourceManager extends CompositeService
}
}
+ /**
+ * Deletes the YarnConfigurationStore.
+ *
+ * @param conf
+ * @throws Exception
+ */
+ @VisibleForTesting
+ static void deleteRMConfStore(Configuration conf) throws Exception {
+ ResourceManager rm = new ResourceManager();
+ rm.conf = conf;
+ ResourceScheduler scheduler = rm.createScheduler();
+ RMContextImpl rmContext = new RMContextImpl();
+ rmContext.setResourceManager(rm);
+
+ boolean isConfigurationMutable = false;
+ String confProviderStr = conf.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.DEFAULT_CONFIGURATION_STORE);
+ switch (confProviderStr) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ isConfigurationMutable = true;
+ break;
+ default:
+ }
+
+ if (scheduler instanceof MutableConfScheduler && isConfigurationMutable) {
+ YarnConfigurationStore confStore = YarnConfigurationStoreFactory
+ .getStore(conf);
+ confStore.initialize(conf, conf, rmContext);
+ confStore.format();
+ } else {
+ System.out.println("Scheduler Configuration format only " +
+ "supported by MutableConfScheduler.");
+ }
+ }
+
@VisibleForTesting
static void removeApplication(Configuration conf, String applicationId)
throws Exception {
@@ -1603,7 +1646,9 @@ public class ResourceManager extends CompositeService
private static void printUsage(PrintStream out) {
out.println("Usage: yarn resourcemanager [-format-state-store]");
out.println(" "
- + "[-remove-application-from-state-store <appId>]" + "\n");
+ + "[-remove-application-from-state-store <appId>]");
+ out.println(" "
+ + "[-format-conf-store]" + "\n");
}
protected RMAppLifetimeMonitor createRMAppLifetimeMonitor() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index fd03d87..ba8a09f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -70,23 +70,7 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
@Override
public void init(Configuration config) throws IOException {
- String store = config.get(
- YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
- YarnConfiguration.MEMORY_CONFIGURATION_STORE);
- switch (store) {
- case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
- this.confStore = new InMemoryConfigurationStore();
- break;
- case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
- this.confStore = new LeveldbConfigurationStore();
- break;
- case YarnConfiguration.ZK_CONFIGURATION_STORE:
- this.confStore = new ZKConfigurationStore();
- break;
- default:
- this.confStore = YarnConfigurationStoreFactory.getStore(config);
- break;
- }
+ this.confStore = YarnConfigurationStoreFactory.getStore(config);
Configuration initialSchedConf = new Configuration(false);
initialSchedConf.addResource(YarnConfiguration.CS_CONFIGURATION_FILE);
this.schedConf = new Configuration(false);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
index 60249c8..9b6893c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStoreFactory.java
@@ -37,10 +37,22 @@ public final class YarnConfigurationStoreFactory {
}
public static YarnConfigurationStore getStore(Configuration conf) {
- Class<? extends YarnConfigurationStore> storeClass =
- conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
- InMemoryConfigurationStore.class, YarnConfigurationStore.class);
- LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
- return ReflectionUtils.newInstance(storeClass, conf);
+ String store = conf.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.MEMORY_CONFIGURATION_STORE);
+ switch (store) {
+ case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
+ return new InMemoryConfigurationStore();
+ case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
+ return new LeveldbConfigurationStore();
+ case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ return new ZKConfigurationStore();
+ default:
+ Class<? extends YarnConfigurationStore> storeClass =
+ conf.getClass(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ InMemoryConfigurationStore.class, YarnConfigurationStore.class);
+ LOG.info("Using YarnConfigurationStore implementation - " + storeClass);
+ return ReflectionUtils.newInstance(storeClass, conf);
+ }
}
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
index 0203351..e2c7834 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMStoreCommands.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -67,6 +69,39 @@ public class TestRMStoreCommands {
}
@Test
+ public void testFormatConfStoreCmdForZK() throws Exception {
+ try (TestingServer curatorTestingServer =
+ TestZKRMStateStore.setupCuratorServer();
+ CuratorFramework curatorFramework = TestZKRMStateStore.
+ setupCuratorFramework(curatorTestingServer)) {
+ Configuration conf = TestZKRMStateStore.createHARMConf("rm1,rm2", "rm1",
+ 1234, false, curatorTestingServer);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.ZK_CONFIGURATION_STORE);
+
+ ResourceManager rm = new MockRM(conf);
+ rm.start();
+
+ String confStorePath = conf.get(
+ YarnConfiguration.RM_SCHEDCONF_STORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_RM_SCHEDCONF_STORE_ZK_PARENT_PATH)
+ + "/CONF_STORE";
+ assertNotNull("Failed to initialize ZKConfigurationStore",
+ curatorFramework.checkExists().forPath(confStorePath));
+
+ rm.close();
+ try {
+ ResourceManager.deleteRMConfStore(conf);
+ } catch (Exception e) {
+ fail("Exception should not be thrown during format rm conf store" +
+ " operation.");
+ }
+ assertNull("Failed to format ZKConfigurationStore",
+ curatorFramework.checkExists().forPath(confStorePath));
+ }
+ }
+
+ @Test
public void testRemoveApplicationFromStateStoreCmdForZK() throws Exception {
StateChangeRequestInfo req = new StateChangeRequestInfo(
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
index e17538c..cfa1ce6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YarnCommands.md
@@ -193,7 +193,7 @@ Usage: `yarn resourcemanager [-format-state-store]`
|:---- |:---- |
| -format-state-store | Formats the RMStateStore. This will clear the RMStateStore and is useful if past applications are no longer needed. This should be run only when the ResourceManager is not running. |
| -remove-application-from-state-store \<appId\> | Remove the application from RMStateStore. This should be run only when the ResourceManager is not running. |
-
+| -format-conf-store | Formats the YarnConfigurationStore. This will clear the persisted scheduler configuration under YarnConfigurationStore. This should be run only when the ResourceManager is not running. |
Start the ResourceManager
### `rmadmin`
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org