You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by va...@apache.org on 2019/09/13 11:34:13 UTC
[hadoop] branch branch-2 updated: YARN-9825. Changes for
initializing placement rules with ResourceScheduler in branch-2.
Contributed by Jonathan Hung.
This is an automated email from the ASF dual-hosted git repository.
varunsaxena pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new c9a4630 YARN-9825. Changes for initializing placement rules with ResourceScheduler in branch-2. Contributed by Jonathan Hung.
c9a4630 is described below
commit c9a46308a1bfac9278b19e179bf3132603d52e99
Author: Varun Saxena <va...@apache.org>
AuthorDate: Fri Sep 13 17:03:47 2019 +0530
YARN-9825. Changes for initializing placement rules with ResourceScheduler in branch-2. Contributed by Jonathan Hung.
---
.../resourcemanager/placement/PlacementRule.java | 9 ++--
.../placement/UserGroupMappingPlacementRule.java | 62 +++++++++++++++++++++-
.../scheduler/capacity/CapacityScheduler.java | 44 +++++----------
3 files changed, 78 insertions(+), 37 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
index 47dc48a..805fcf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
@@ -18,20 +18,19 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
-import java.util.Map;
+import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
public abstract class PlacementRule {
public String getName() {
return this.getClass().getName();
}
- public void initialize(Map<String, String> parameters, RMContext rmContext)
- throws YarnException {
- }
+ public abstract boolean initialize(
+ ResourceScheduler scheduler) throws IOException;
/**
* Get queue for a given application
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index d617d16..bac419d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -32,6 +32,14 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+
public class UserGroupMappingPlacementRule extends PlacementRule {
private static final Log LOG = LogFactory
@@ -95,6 +103,10 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
}
}
+ public UserGroupMappingPlacementRule(){
+ this(false, null, null);
+ }
+
public UserGroupMappingPlacementRule(boolean overrideWithQueueMappings,
List<QueueMapping> newMappings, Groups groups) {
this.mappings = newMappings;
@@ -156,7 +168,55 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
return queueName;
}
-
+
+ @VisibleForTesting
+ @Override
+ public boolean initialize(ResourceScheduler scheduler)
+ throws IOException {
+ if (!(scheduler instanceof CapacityScheduler)) {
+ throw new IOException(
+ "UserGroupMappingPlacementRule can be configured only for "
+ + "CapacityScheduler");
+ }
+ CapacitySchedulerContext schedulerContext =
+ (CapacitySchedulerContext) scheduler;
+ CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
+ boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
+ LOG.info(
+ "Initialized queue mappings, override: " + overrideWithQueueMappings);
+
+ // Get new user/group mappings
+ List<QueueMapping> newMappings = conf.getQueueMappings();
+
+ CapacitySchedulerQueueManager queueManager =
+ schedulerContext.getCapacitySchedulerQueueManager();
+
+ // check if mappings refer to valid queues
+ for (QueueMapping mapping : newMappings) {
+ String mappingQueue = mapping.getQueue();
+ if (!mappingQueue.equals(
+ UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mappingQueue
+ .equals(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)) {
+ CSQueue queue = queueManager.getQueue(mappingQueue);
+ if (queue == null || !(queue instanceof LeafQueue)) {
+ throw new IOException(
+ "mapping contains invalid or non-leaf queue " + mappingQueue);
+ }
+ }
+ }
+
+ // initialize groups if mappings are present
+ if (newMappings.size() > 0) {
+ Groups groups = new Groups(conf);
+ this.mappings = newMappings;
+ this.groups = groups;
+ this.overrideWithQueueMappings = overrideWithQueueMappings;
+ return true;
+ }
+
+ return false;
+ }
+
@VisibleForTesting
public List<QueueMapping> getQueueMappings() {
return mappings;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index a19c47c..9ec553e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -569,38 +569,12 @@ public class CapacityScheduler extends
}
@VisibleForTesting
- public UserGroupMappingPlacementRule
- getUserGroupMappingPlacementRule() throws IOException {
+ public PlacementRule getUserGroupMappingPlacementRule() throws IOException {
try {
readLock.lock();
- boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
- LOG.info(
- "Initialized queue mappings, override: " + overrideWithQueueMappings);
-
- // Get new user/group mappings
- List<QueueMapping> newMappings = conf.getQueueMappings();
- // check if mappings refer to valid queues
- for (QueueMapping mapping : newMappings) {
- String mappingQueue = mapping.getQueue();
- if (!mappingQueue.equals(
- UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mappingQueue
- .equals(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)) {
- CSQueue queue = getQueue(mappingQueue);
- if (queue == null || !(queue instanceof LeafQueue)) {
- throw new IOException(
- "mapping contains invalid or non-leaf queue " + mappingQueue);
- }
- }
- }
-
- // initialize groups if mappings are present
- if (newMappings.size() > 0) {
- Groups groups = new Groups(conf);
- return new UserGroupMappingPlacementRule(overrideWithQueueMappings,
- newMappings, groups);
- }
-
- return null;
+ UserGroupMappingPlacementRule ugRule = new UserGroupMappingPlacementRule();
+ ugRule.initialize(this);
+ return ugRule;
} finally {
readLock.unlock();
}
@@ -626,11 +600,19 @@ public class CapacityScheduler extends
}
break;
default:
+ boolean isMappingNotEmpty;
try {
PlacementRule rule = PlacementFactory.getPlacementRule(
placementRuleStr, conf);
if (null != rule) {
- placementRules.add(rule);
+ try {
+ isMappingNotEmpty = rule.initialize(this);
+ } catch (IOException ie) {
+ throw new IOException(ie);
+ }
+ if (isMappingNotEmpty) {
+ placementRules.add(rule);
+ }
}
} catch (ClassNotFoundException cnfe) {
throw new IOException(cnfe);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org