You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/07/19 21:41:40 UTC
[40/50] hadoop git commit: YARN-5272. Handle queue names consistently
in FairScheduler. (Wilfred Spiegelenburg via rchiang)
YARN-5272. Handle queue names consistently in FairScheduler. (Wilfred Spiegelenburg via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46e39856
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46e39856
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46e39856
Branch: refs/heads/YARN-2915
Commit: 46e39856204ad4db42050de191139a233b74a921
Parents: f878bd4
Author: Ray Chiang <rc...@apache.org>
Authored: Fri Jul 15 14:35:50 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Jul 19 14:39:32 2016 -0700
----------------------------------------------------------------------
.../fair/AllocationFileLoaderService.java | 4 ++-
.../scheduler/fair/QueueManager.java | 6 ++++-
.../fair/TestAllocationFileLoaderService.java | 28 ++++++++++++++++++++
.../scheduler/fair/TestFairScheduler.java | 11 ++++++++
.../scheduler/fair/TestQueueManager.java | 3 +++
5 files changed, 50 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
index 661caa7..fab536d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationFileLoaderService.java
@@ -53,6 +53,7 @@ import org.w3c.dom.NodeList;
import org.w3c.dom.Text;
import org.xml.sax.SAXException;
+import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
@Public
@@ -445,7 +446,8 @@ public class AllocationFileLoaderService extends AbstractService {
Set<String> reservableQueues,
Set<String> nonPreemptableQueues)
throws AllocationConfigurationException {
- String queueName = element.getAttribute("name").trim();
+ String queueName = CharMatcher.WHITESPACE.trimFrom(
+ element.getAttribute("name"));
if (queueName.contains(".")) {
throw new AllocationConfigurationException("Bad fair scheduler config "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index aeadcf6..8d308dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.xml.sax.SAXException;
+import com.google.common.base.CharMatcher;
import com.google.common.annotations.VisibleForTesting;
/**
* Maintains a list of queues as well as scheduling parameters for each queue,
@@ -457,6 +458,9 @@ public class QueueManager {
*/
@VisibleForTesting
boolean isQueueNameValid(String node) {
- return !node.isEmpty() && node.equals(node.trim());
+ // use the same white space trim as in QueueMetrics() otherwise things fail
+ // guava uses a different definition for whitespace than java.
+ return !node.isEmpty() &&
+ node.equals(CharMatcher.WHITESPACE.trimFrom(node));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index cc91ef9..11d4981 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -20,8 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.*;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.FileWriter;
+import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.nio.charset.StandardCharsets;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -581,6 +584,31 @@ public class TestAllocationFileLoaderService {
}
/**
+ * Verify that you can't have the queue name with just a non breaking
+ * whitespace in the allocations file.
+ */
+ @Test (expected = AllocationConfigurationException.class)
+ public void testQueueNameContainingNBWhitespace() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new OutputStreamWriter(
+ new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8));
+ out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"\u00a0\">");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+ allocLoader.init(conf);
+ ReloadListener confHolder = new ReloadListener();
+ allocLoader.setReloadListener(confHolder);
+ allocLoader.reloadAllocations();
+ }
+
+ /**
* Verify that defaultQueueSchedulingMode can't accept FIFO as a value.
*/
@Test (expected = AllocationConfigurationException.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 43ebe53..f92af35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4362,6 +4362,17 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals(1, scheduler.getQueueManager().getLeafQueue("B.C", true)
.getNumRunnableApps());
assertNotNull(scheduler.getSchedulerApp(appAttemptId3));
+
+ // submit app with queue name "A\u00a0" (non-breaking space)
+ ApplicationAttemptId appAttemptId4 = createAppAttemptId(4, 1);
+ AppAddedSchedulerEvent appAddedEvent4 = new AppAddedSchedulerEvent(
+ appAttemptId4.getApplicationId(), "A\u00a0", "user1");
+ scheduler.handle(appAddedEvent4);
+ // submission rejected
+ assertEquals(3, scheduler.getQueueManager().getLeafQueues().size());
+ assertNull(scheduler.getSchedulerApplications().get(appAttemptId4.
+ getApplicationId()));
+ assertNull(scheduler.getSchedulerApp(appAttemptId4));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46e39856/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index a9b27a1..33d4419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -131,6 +131,9 @@ public class TestQueueManager {
assertFalse(queueManager.isQueueNameValid(" a"));
assertFalse(queueManager.isQueueNameValid("a "));
assertFalse(queueManager.isQueueNameValid(" a "));
+ assertFalse(queueManager.isQueueNameValid("\u00a0"));
+ assertFalse(queueManager.isQueueNameValid("a\u00a0"));
+ assertFalse(queueManager.isQueueNameValid("\u00a0a\u00a0"));
assertTrue(queueManager.isQueueNameValid("a b"));
assertTrue(queueManager.isQueueNameValid("a"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org