You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/05/12 02:31:37 UTC
hadoop git commit: YARN-3489. RMServerUtils.validateResourceRequests
should only obtain queue info once. (Varun Saxena via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 444836b3d -> d6f674129
YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6f67412
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6f67412
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6f67412
Branch: refs/heads/trunk
Commit: d6f6741296639a73f5306e3ebefec84a40ca03e5
Parents: 444836b
Author: Wangda Tan <wa...@apache.org>
Authored: Mon May 11 17:31:15 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon May 11 17:31:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/RMServerUtils.java | 10 ++++++-
.../scheduler/SchedulerUtils.java | 30 +++++++++++++++-----
3 files changed, 35 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6f67412/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 80cdf35..b5cb0a5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -405,6 +405,9 @@ Release 2.7.1 - UNRELEASED
YARN-3243. CapacityScheduler should pass headroom from parent to children
to make sure ParentQueue obey its capacity limits. (Wangda Tan via jianhe)
+ YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue
+ info once. (Varun Saxena via wangda)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6f67412/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 4669a28..4d2e41c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -94,9 +95,16 @@ public class RMServerUtils {
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext)
throws InvalidResourceRequestException {
+ // Get queue from scheduler
+ QueueInfo queueInfo = null;
+ try {
+ queueInfo = scheduler.getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ }
+
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
- queueName, scheduler, rmContext);
+ queueName, scheduler, rmContext, queueInfo);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6f67412/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 0ef5c1e..8047d0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -213,13 +213,21 @@ public class SchedulerUtils {
Resource maximumResource, String queueName, YarnScheduler scheduler,
boolean isRecovery, RMContext rmContext)
throws InvalidResourceRequestException {
+ normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
+ isRecovery, rmContext, null);
+ }
- QueueInfo queueInfo = null;
- try {
- queueInfo = scheduler.getQueueInfo(queueName, false, false);
- } catch (IOException e) {
- // it is possible queue cannot get when queue mapping is set, just ignore
- // the queueInfo here, and move forward
+ public static void normalizeAndValidateRequest(ResourceRequest resReq,
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
+ throws InvalidResourceRequestException {
+ if (null == queueInfo) {
+ try {
+ queueInfo = scheduler.getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ // it is possible queue cannot get when queue mapping is set, just ignore
+ // the queueInfo here, and move forward
+ }
}
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
@@ -231,8 +239,16 @@ public class SchedulerUtils {
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext)
throws InvalidResourceRequestException {
+ normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler,
+ rmContext, null);
+ }
+
+ public static void normalizeAndvalidateRequest(ResourceRequest resReq,
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext, QueueInfo queueInfo)
+ throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
- false, rmContext);
+ false, rmContext, queueInfo);
}
/**