You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/05/12 22:49:44 UTC
[27/36] hadoop git commit: YARN-3489.
RMServerUtils.validateResourceRequests should only obtain queue info once.
(Varun Saxena via wangda)
YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue info once. (Varun Saxena via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca82dc48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca82dc48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca82dc48
Branch: refs/heads/YARN-2928
Commit: ca82dc4858ee6e12fffe8924ec3f242713b4aeb8
Parents: 5625ac4
Author: Wangda Tan <wa...@apache.org>
Authored: Mon May 11 17:31:15 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue May 12 13:44:26 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/RMServerUtils.java | 10 ++++++-
.../scheduler/SchedulerUtils.java | 30 +++++++++++++++-----
3 files changed, 35 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca82dc48/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0a1b853..e02a564 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -478,6 +478,9 @@ Release 2.7.1 - UNRELEASED
YARN-3243. CapacityScheduler should pass headroom from parent to children
to make sure ParentQueue obey its capacity limits. (Wangda Tan via jianhe)
+ YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue
+ info once. (Varun Saxena via wangda)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca82dc48/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 4669a28..4d2e41c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -94,9 +95,16 @@ public class RMServerUtils {
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext)
throws InvalidResourceRequestException {
+ // Get queue from scheduler
+ QueueInfo queueInfo = null;
+ try {
+ queueInfo = scheduler.getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ }
+
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
- queueName, scheduler, rmContext);
+ queueName, scheduler, rmContext, queueInfo);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca82dc48/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 0ef5c1e..8047d0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -213,13 +213,21 @@ public class SchedulerUtils {
Resource maximumResource, String queueName, YarnScheduler scheduler,
boolean isRecovery, RMContext rmContext)
throws InvalidResourceRequestException {
+ normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
+ isRecovery, rmContext, null);
+ }
- QueueInfo queueInfo = null;
- try {
- queueInfo = scheduler.getQueueInfo(queueName, false, false);
- } catch (IOException e) {
- // it is possible queue cannot get when queue mapping is set, just ignore
- // the queueInfo here, and move forward
+ public static void normalizeAndValidateRequest(ResourceRequest resReq,
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ boolean isRecovery, RMContext rmContext, QueueInfo queueInfo)
+ throws InvalidResourceRequestException {
+ if (null == queueInfo) {
+ try {
+ queueInfo = scheduler.getQueueInfo(queueName, false, false);
+ } catch (IOException e) {
+ // it is possible queue cannot get when queue mapping is set, just ignore
+ // the queueInfo here, and move forward
+ }
}
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
@@ -231,8 +239,16 @@ public class SchedulerUtils {
Resource maximumResource, String queueName, YarnScheduler scheduler,
RMContext rmContext)
throws InvalidResourceRequestException {
+ normalizeAndvalidateRequest(resReq, maximumResource, queueName, scheduler,
+ rmContext, null);
+ }
+
+ public static void normalizeAndvalidateRequest(ResourceRequest resReq,
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext, QueueInfo queueInfo)
+ throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
- false, rmContext);
+ false, rmContext, queueInfo);
}
/**