You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/02/03 20:08:27 UTC
svn commit: r1564003 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/
hadoop-yarn/hadoop-yarn-commo...
Author: vinodkv
Date: Mon Feb 3 19:08:26 2014
New Revision: 1564003
URL: http://svn.apache.org/r1564003
Log:
YARN-1611. Introduced the concept of a configuration provider which can be used by ResourceManager to read configuration locally or from remote systems so as to help RM failover. Contributed by Xuan Gong.
svn merge -c 1564002 ../../trunk/
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
- copied unchanged from r1564002, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProvider.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
- copied unchanged from r1564002, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/ConfigurationProviderFactory.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
- copied unchanged from r1564002, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/FileSystemBasedConfigurationProvider.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
- copied unchanged from r1564002, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/LocalConfigurationProvider.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
- copied unchanged from r1564002, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/ (props changed)
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
Propchange: hadoop/common/branches/branch-2/hadoop-yarn-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-yarn-project:r1564002
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1564003&r1=1564002&r2=1564003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Feb 3 19:08:26 2014
@@ -59,6 +59,10 @@ Release 2.4.0 - UNRELEASED
YARN-1633. Defined user-facing entity, entity-info and event objects related
to Application Timeline feature. (Zhijie Shen via vinodkv)
+ YARN-1611. Introduced the concept of a configuration provider which can be
+ used by ResourceManager to read configuration locally or from remote systems
+ so as to help RM failover. (Xuan Gong via vinodkv)
+
IMPROVEMENTS
YARN-1007. Enhance History Reader interface for Containers. (Mayank Bansal via
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1564003&r1=1564002&r2=1564003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Mon Feb 3 19:08:26 2014
@@ -37,6 +37,9 @@ import org.apache.hadoop.yarn.api.Applic
@Evolving
public class YarnConfiguration extends Configuration {
+ @Private
+ public static final String CS_CONFIGURATION_FILE= "capacity-scheduler.xml";
+
private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
private static final String YARN_SITE_XML_FILE = "yarn-site.xml";
@@ -329,6 +332,16 @@ public class YarnConfiguration extends C
public static final String RM_HA_IDS = RM_HA_PREFIX + "rm-ids";
public static final String RM_HA_ID = RM_HA_PREFIX + "id";
+ /** Store the related configuration files in File System */
+ public static final String FS_BASED_RM_CONF_STORE = RM_PREFIX
+ + "configuration.file-system-based-store";
+ public static final String DEFAULT_FS_BASED_RM_CONF_STORE = "/yarn/conf";
+
+ public static final String RM_CONFIGURATION_PROVIDER_CLASS = RM_PREFIX
+ + "configuration.provider-class";
+ public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
+ "org.apache.hadoop.yarn.LocalConfigurationProvider";
+
@Private
public static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS =
Collections.unmodifiableList(Arrays.asList(
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1564003&r1=1564002&r2=1564003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Mon Feb 3 19:08:26 2014
@@ -588,6 +588,18 @@
<value>org.apache.hadoop.yarn.server.applicationhistoryservice.NullApplicationHistoryStore</value>
</property>
+ <property>
+ <description>The class to use as the configuration provider.
+ If org.apache.hadoop.yarn.LocalConfigurationProvider is used,
+ the local configuration will be loaded.
+ If org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider is used,
+ the configuration which will be loaded should be uploaded to remote File system first.
+ </description>>
+ <name>yarn.resourcemanager.configuration.provider-class</name>
+ <value>org.apache.hadoop.yarn.LocalConfigurationProvider</value>
+ <!-- <value>org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider</value> -->
+ </property>
+
<!-- Node Manager Configs -->
<property>
<description>The hostname of the NM.</description>
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1564003&r1=1564002&r2=1564003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Mon Feb 3 19:08:26 2014
@@ -47,6 +47,8 @@ import org.apache.hadoop.security.author
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
+import org.apache.hadoop.yarn.conf.ConfigurationProvider;
+import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -89,6 +91,8 @@ public class AdminService extends Compos
private InetSocketAddress masterServiceAddress;
private AccessControlList adminAcl;
+ private ConfigurationProvider configurationProvider = null;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -109,6 +113,10 @@ public class AdminService extends Compos
}
}
+ this.configurationProvider =
+ ConfigurationProviderFactory.getConfigurationProvider(conf);
+ configurationProvider.init(conf);
+
masterServiceAddress = conf.getSocketAddr(
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
@@ -129,6 +137,9 @@ public class AdminService extends Compos
@Override
protected synchronized void serviceStop() throws Exception {
stopServer();
+ if (this.configurationProvider != null) {
+ configurationProvider.close();
+ }
super.serviceStop();
}
@@ -295,23 +306,28 @@ public class AdminService extends Compos
@Override
public RefreshQueuesResponse refreshQueues(RefreshQueuesRequest request)
throws YarnException, StandbyException {
- UserGroupInformation user = checkAcls("refreshQueues");
+ String argName = "refreshQueues";
+ UserGroupInformation user = checkAcls(argName);
if (!isRMActive()) {
- RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"ResourceManager is not active. Can not refresh queues.");
throwStandbyException();
}
+ RefreshQueuesResponse response =
+ recordFactory.newRecordInstance(RefreshQueuesResponse.class);
try {
- rmContext.getScheduler().reinitialize(getConfig(), this.rmContext);
- RMAuditLogger.logSuccess(user.getShortUserName(), "refreshQueues",
+ Configuration conf =
+ getConfiguration(YarnConfiguration.CS_CONFIGURATION_FILE);
+ rmContext.getScheduler().reinitialize(conf, this.rmContext);
+ RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
- return recordFactory.newRecordInstance(RefreshQueuesResponse.class);
+ return response;
} catch (IOException ioe) {
LOG.info("Exception refreshing queues ", ioe);
- RMAuditLogger.logFailure(user.getShortUserName(), "refreshQueues",
+ RMAuditLogger.logFailure(user.getShortUserName(), argName,
adminAcl.toString(), "AdminService",
"Exception refreshing queues");
throw RPCUtil.getRemoteException(ioe);
@@ -483,5 +499,9 @@ public class AdminService extends Compos
UpdateNodeResourceResponse.class);
return response;
}
-
+
+ private synchronized Configuration getConfiguration(String confFileName)
+ throws YarnException, IOException {
+ return this.configurationProvider.getConfiguration(confFileName);
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1564003&r1=1564002&r2=1564003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Mon Feb 3 19:08:26 2014
@@ -195,6 +195,7 @@ public class CapacityScheduler extends A
private ResourceCalculator calculator;
private boolean usePortForNodeName;
+ private boolean useLocalConfigurationProvider;
public CapacityScheduler() {}
@@ -261,7 +262,13 @@ public class CapacityScheduler extends A
public synchronized void
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
if (!initialized) {
- this.conf = new CapacitySchedulerConfiguration(conf);
+ this.useLocalConfigurationProvider = conf.get(
+ YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
+ YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS).equals(
+ "org.apache.hadoop.yarn.LocalConfigurationProvider");
+ this.conf =
+ new CapacitySchedulerConfiguration(conf,
+ this.useLocalConfigurationProvider);
validateConf(this.conf);
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
@@ -279,9 +286,10 @@ public class CapacityScheduler extends A
"minimumAllocation=<" + getMinimumResourceCapability() + ">, " +
"maximumAllocation=<" + getMaximumResourceCapability() + ">");
} else {
-
CapacitySchedulerConfiguration oldConf = this.conf;
- this.conf = new CapacitySchedulerConfiguration(conf);
+ this.conf =
+ new CapacitySchedulerConfiguration(conf,
+ this.useLocalConfigurationProvider);
validateConf(this.conf);
try {
LOG.info("Re-initializing queues...");
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1564003&r1=1564002&r2=1564003&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Mon Feb 3 19:08:26 2014
@@ -140,10 +140,17 @@ public class CapacitySchedulerConfigurat
}
public CapacitySchedulerConfiguration(Configuration configuration) {
+ this(configuration, true);
+ }
+
+ public CapacitySchedulerConfiguration(Configuration configuration,
+ boolean useLocalConfigurationProvider) {
super(configuration);
- addResource(CS_CONFIGURATION_FILE);
+ if (useLocalConfigurationProvider) {
+ addResource(CS_CONFIGURATION_FILE);
+ }
}
-
+
private String getQueuePrefix(String queue) {
String queueName = PREFIX + queue + DOT;
return queueName;