You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2016/01/06 15:03:05 UTC
[1/2] hadoop git commit: YARN-4546. ResourceManager crash due to
scheduling opportunity overflow. Contributed by Jason Lowe. (cherry picked
from commit c1462a67ff7bb632df50e1c52de971cced56c6a3) (cherry picked from
commit 1cc001db4c3767072b5d065d161bc5c6d
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 b2d02863a -> 74027c24c
YARN-4546. ResourceManager crash due to scheduling opportunity overflow. Contributed by Jason Lowe.
(cherry picked from commit c1462a67ff7bb632df50e1c52de971cced56c6a3)
(cherry picked from commit 1cc001db4c3767072b5d065d161bc5c6d1c480d4)
Conflicts:
hadoop-yarn-project/CHANGES.txt
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0ffe25a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0ffe25a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0ffe25a
Branch: refs/heads/branch-2.6
Commit: c0ffe25a6547dbb069993f87e20d9c723c4686dc
Parents: b2d0286
Author: Junping Du <ju...@apache.org>
Authored: Wed Jan 6 05:49:24 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Wed Jan 6 05:56:16 2016 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../scheduler/SchedulerApplicationAttempt.java | 11 ++++++++--
.../TestSchedulerApplicationAttempt.java | 22 ++++++++++++++++++++
3 files changed, 34 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ffe25a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 38089a1..6675dce 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -28,6 +28,9 @@ Release 2.6.4 - UNRELEASED
YARN-3697. FairScheduler: ContinuousSchedulingThread can fail to shutdown.
(Zhihai Xu via kasha)
+ YARN-4546. ResourceManager crash due to scheduling opportunity overflow.
+ (Jason Lowe via junping_du)
+
Release 2.6.3 - 2015-12-17
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ffe25a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 510276c..5629734 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -482,8 +482,10 @@ public class SchedulerApplicationAttempt {
}
public synchronized void addSchedulingOpportunity(Priority priority) {
- schedulingOpportunities.setCount(priority,
- schedulingOpportunities.count(priority) + 1);
+ int count = schedulingOpportunities.count(priority);
+ if (count < Integer.MAX_VALUE) {
+ schedulingOpportunities.setCount(priority, count + 1);
+ }
}
public synchronized void subtractSchedulingOpportunity(Priority priority) {
@@ -517,6 +519,11 @@ public class SchedulerApplicationAttempt {
schedulingOpportunities.setCount(priority, 0);
}
+ @VisibleForTesting
+ void setSchedulingOpportunities(Priority priority, int count) {
+ schedulingOpportunities.setCount(priority, count);
+ }
+
synchronized AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
long currentTimeMillis = System.currentTimeMillis();
// Don't walk the whole container list if the resources were computed
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0ffe25a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index c648b83..973c0b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -163,4 +163,26 @@ public class TestSchedulerApplicationAttempt {
ApplicationAttemptId.newInstance(appIdImpl, attemptId);
return attId;
}
+
+ @Test
+ public void testSchedulingOpportunityOverflow() throws Exception {
+ ApplicationAttemptId attemptId = createAppAttemptId(0, 0);
+ Queue queue = createQueue("test", null);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(
+ attemptId, "user", queue, queue.getActiveUsersManager(), rmContext);
+ Priority priority = Priority.newInstance(1);
+ assertEquals(0, app.getSchedulingOpportunities(priority));
+ app.addSchedulingOpportunity(priority);
+ assertEquals(1, app.getSchedulingOpportunities(priority));
+ // verify the count is capped at MAX_VALUE and does not overflow
+ app.setSchedulingOpportunities(priority, Integer.MAX_VALUE - 1);
+ assertEquals(Integer.MAX_VALUE - 1,
+ app.getSchedulingOpportunities(priority));
+ app.addSchedulingOpportunity(priority);
+ assertEquals(Integer.MAX_VALUE, app.getSchedulingOpportunities(priority));
+ app.addSchedulingOpportunity(priority);
+ assertEquals(Integer.MAX_VALUE, app.getSchedulingOpportunities(priority));
+ }
}
[2/2] hadoop git commit: Addendum patch to fix build after porting
YARN-4546.
Posted by ju...@apache.org.
Addendum patch to fix build after porting YARN-4546.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74027c24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74027c24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74027c24
Branch: refs/heads/branch-2.6
Commit: 74027c24c810c641712c63199c48c9497bdd9273
Parents: c0ffe25
Author: Junping Du <ju...@apache.org>
Authored: Wed Jan 6 06:11:00 2016 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Wed Jan 6 06:11:00 2016 -0800
----------------------------------------------------------------------
.../resourcemanager/scheduler/SchedulerApplicationAttempt.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74027c24/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 5629734..b0c97a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;