You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/07/12 22:42:56 UTC
hadoop git commit: YARN-6689. PlacementRule should be configurable.
(Jonathan Hung via xgong)
Repository: hadoop
Updated Branches:
refs/heads/trunk 931a49800 -> 9845bea59
YARN-6689. PlacementRule should be configurable. (Jonathan Hung via xgong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9845bea5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9845bea5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9845bea5
Branch: refs/heads/trunk
Commit: 9845bea59c0009b8ffff7050403e5ba270054164
Parents: 931a498
Author: Xuan <xg...@apache.org>
Authored: Wed Jul 12 15:40:45 2017 -0700
Committer: Xuan <xg...@apache.org>
Committed: Wed Jul 12 15:40:45 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +++
.../src/main/resources/yarn-default.xml | 10 +++++
.../placement/PlacementFactory.java | 45 ++++++++++++++++++++
.../scheduler/capacity/CapacityScheduler.java | 37 +++++++++++++---
4 files changed, 91 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9845bea5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a6d3360..01eff64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -210,6 +210,12 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME =
false;
+ /** Configured scheduler queue placement rules. */
+ public static final String QUEUE_PLACEMENT_RULES = YARN_PREFIX
+ + "scheduler.queue-placement-rules";
+ /** UserGroupMappingPlacementRule configuration string. */
+ public static final String USER_GROUP_PLACEMENT_RULE = "user-group";
+
/** Enable Resource Manager webapp ui actions */
public static final String RM_WEBAPP_UI_ACTIONS_ENABLED =
RM_PREFIX + "webapp.ui-actions.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9845bea5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 81c9cb2..0588c6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3116,4 +3116,14 @@
<value>false</value>
</property>
+ <property>
+ <description>
+ Comma-separated list of PlacementRules to determine how applications
+ submitted by certain users get mapped to certain queues. Default is
+ user-group, which corresponds to UserGroupMappingPlacementRule.
+ </description>
+ <name>yarn.scheduler.queue-placement-rules</name>
+ <value>user-group</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9845bea5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java
new file mode 100644
index 0000000..a5de664
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementFactory.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * Factory class for creating instances of {@link PlacementRule}.
+ */
+public final class PlacementFactory {
+
+ private static final Log LOG = LogFactory.getLog(PlacementFactory.class);
+
+ private PlacementFactory() {
+ // Unused.
+ }
+
+ public static PlacementRule getPlacementRule(String ruleStr,
+ Configuration conf)
+ throws ClassNotFoundException {
+ Class<? extends PlacementRule> ruleClass = Class.forName(ruleStr)
+ .asSubclass(PlacementRule.class);
+ LOG.info("Using PlacementRule implementation - " + ruleClass);
+ return ReflectionUtils.newInstance(ruleClass, conf);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9845bea5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 0d72860..2ccaf63 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementFactory;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
@@ -564,15 +565,37 @@ public class CapacityScheduler extends
}
private void updatePlacementRules() throws IOException {
+ // Initialize placement rules
+ Collection<String> placementRuleStrs = conf.getStringCollection(
+ YarnConfiguration.QUEUE_PLACEMENT_RULES);
List<PlacementRule> placementRules = new ArrayList<>();
-
- // Initialize UserGroupMappingPlacementRule
- // TODO, need make this defineable by configuration.
- UserGroupMappingPlacementRule ugRule = getUserGroupMappingPlacementRule();
- if (null != ugRule) {
- placementRules.add(ugRule);
+ if (placementRuleStrs.isEmpty()) {
+ PlacementRule ugRule = getUserGroupMappingPlacementRule();
+ if (null != ugRule) {
+ placementRules.add(ugRule);
+ }
+ } else {
+ for (String placementRuleStr : placementRuleStrs) {
+ switch (placementRuleStr) {
+ case YarnConfiguration.USER_GROUP_PLACEMENT_RULE:
+ PlacementRule ugRule = getUserGroupMappingPlacementRule();
+ if (null != ugRule) {
+ placementRules.add(ugRule);
+ }
+ break;
+ default:
+ try {
+ PlacementRule rule = PlacementFactory.getPlacementRule(
+ placementRuleStr, conf);
+ if (null != rule) {
+ placementRules.add(rule);
+ }
+ } catch (ClassNotFoundException cnfe) {
+ throw new IOException(cnfe);
+ }
+ }
+ }
}
-
rmContext.getQueuePlacementManager().updateRules(placementRules);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org