You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ad...@apache.org on 2015/09/28 18:35:14 UTC
hadoop git commit: YARN-4204. ConcurrentModificationException in
FairSchedulerQueueInfo. (adhoot)
Repository: hadoop
Updated Branches:
refs/heads/trunk 892ade689 -> fb2e525c0
YARN-4204. ConcurrentModificationException in FairSchedulerQueueInfo. (adhoot)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fb2e525c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fb2e525c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fb2e525c
Branch: refs/heads/trunk
Commit: fb2e525c0775ccf218c8980676e9fb4005a406a6
Parents: 892ade6
Author: Anubhav Dhoot <ad...@apache.org>
Authored: Sun Sep 27 20:52:38 2015 -0700
Committer: Anubhav Dhoot <ad...@apache.org>
Committed: Mon Sep 28 09:05:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../scheduler/fair/FSLeafQueue.java | 5 +-
.../scheduler/fair/FSParentQueue.java | 3 +-
.../scheduler/fair/QueueManager.java | 24 +++---
.../scheduler/fair/TestFSParentQueue.java | 79 ++++++++++++++++++++
5 files changed, 100 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb2e525c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f7ea26e..54207aa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -902,6 +902,8 @@ Release 2.8.0 - UNRELEASED
YARN-4044. Running applications information changes such as movequeue is not published to
TimeLine server. (Sunil G via rohithsharmaks)
+ YARN-4204. ConcurrentModificationException in FairSchedulerQueueInfo. (adhoot)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb2e525c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index f90a198..ca5a146 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -70,7 +70,8 @@ public class FSLeafQueue extends FSQueue {
private Resource amResourceUsage;
private final ActiveUsersManager activeUsersManager;
-
+ public static final List<FSQueue> EMPTY_LIST = Collections.emptyList();
+
public FSLeafQueue(String name, FairScheduler scheduler,
FSParentQueue parent) {
super(name, scheduler, parent);
@@ -383,7 +384,7 @@ public class FSLeafQueue extends FSQueue {
@Override
public List<FSQueue> getChildQueues() {
- return new ArrayList<FSQueue>(1);
+ return EMPTY_LIST;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb2e525c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 7d2e5b8..febe050 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -27,6 +27,7 @@ import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -279,7 +280,7 @@ public class FSParentQueue extends FSQueue {
public List<FSQueue> getChildQueues() {
readLock.lock();
try {
- return Collections.unmodifiableList(childQueues);
+ return ImmutableList.copyOf(childQueues);
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb2e525c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 6556717..0092845 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -28,6 +28,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import javax.xml.parsers.ParserConfigurationException;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -295,17 +296,18 @@ public class QueueManager {
* Remove a queue and all its descendents.
*/
private void removeQueue(FSQueue queue) {
- if (queue instanceof FSLeafQueue) {
- leafQueues.remove(queue);
- } else {
- List<FSQueue> childQueues = queue.getChildQueues();
- while (!childQueues.isEmpty()) {
- removeQueue(childQueues.get(0));
+ synchronized (queues) {
+ if (queue instanceof FSLeafQueue) {
+ leafQueues.remove(queue);
+ } else {
+ for (FSQueue childQueue:queue.getChildQueues()) {
+ removeQueue(childQueue);
+ }
}
+ queues.remove(queue.getName());
+ FSParentQueue parent = queue.getParent();
+ parent.removeChildQueue(queue);
}
- queues.remove(queue.getName());
- FSParentQueue parent = queue.getParent();
- parent.removeChildQueue(queue);
}
/**
@@ -360,7 +362,9 @@ public class QueueManager {
* Get a collection of all queues
*/
public Collection<FSQueue> getQueues() {
- return queues.values();
+ synchronized (queues) {
+ return ImmutableList.copyOf(queues.values());
+ }
}
private String ensureRootPrefix(String name) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fb2e525c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
new file mode 100644
index 0000000..f3e9e0c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSParentQueue.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
+
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestFSParentQueue {
+
+ private FairSchedulerConfiguration conf;
+ private QueueManager queueManager;
+ private Set<FSQueue> notEmptyQueues;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new FairSchedulerConfiguration();
+ FairScheduler scheduler = mock(FairScheduler.class);
+ AllocationConfiguration allocConf = new AllocationConfiguration(conf);
+ when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);
+ when(scheduler.getConf()).thenReturn(conf);
+ SystemClock clock = new SystemClock();
+ when(scheduler.getClock()).thenReturn(clock);
+ notEmptyQueues = new HashSet<FSQueue>();
+ queueManager = new QueueManager(scheduler) {
+ @Override
+ public boolean isEmpty(FSQueue queue) {
+ return !notEmptyQueues.contains(queue);
+ }
+ };
+ FSQueueMetrics.forQueue("root", null, true, conf);
+ queueManager.initialize(conf);
+ }
+
+ @Test
+ public void testConcurrentChangeToGetChildQueue() {
+
+ queueManager.getLeafQueue("parent.child", true);
+ queueManager.getLeafQueue("parent.child2", true);
+ FSParentQueue test = queueManager.getParentQueue("parent", false);
+ assertEquals(2, test.getChildQueues().size());
+
+ boolean first = true;
+ int childQueuesFound = 0;
+ for (FSQueue childQueue:test.getChildQueues()) {
+ if (first) {
+ first = false;
+ queueManager.getLeafQueue("parent.child3", true);
+ }
+ childQueuesFound++;
+ }
+
+ assertEquals(2, childQueuesFound);
+ assertEquals(3, test.getChildQueues().size());
+ }
+}