You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by tg...@apache.org on 2012/08/20 17:36:52 UTC
svn commit: r1375066 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-s...
Author: tgraves
Date: Mon Aug 20 15:36:51 2012
New Revision: 1375066
URL: http://svn.apache.org/viewvc?rev=1375066&view=rev
Log:
YARN-27. Failed refreshQueues due to misconfiguration prevents further refreshing of queues (Arun Murthy via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1375066&r1=1375065&r2=1375066&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Mon Aug 20 15:36:51 2012
@@ -49,3 +49,5 @@ Release 0.23.3 - Unreleased
YARN-25. remove old aggregated logs (Robert Evans via tgraves)
+ YARN-27. Failed refreshQueues due to misconfiguration prevents further
+ refreshing of queues (Arun Murthy via tgraves)
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1375066&r1=1375065&r2=1375066&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Mon Aug 20 15:36:51 2012
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsInfo;
@@ -120,14 +121,41 @@ public class QueueMetrics implements Met
enableUserMetrics, conf);
}
- public static QueueMetrics forQueue(MetricsSystem ms, String queueName,
+ /**
+ * Helper method to clear cache - used only for unit tests.
+ */
+ @Private
+ public synchronized static void clearQueueMetrics() {
+ queueMetrics.clear();
+ }
+
+ /**
+ * Simple metrics cache to help prevent re-registrations.
+ */
+ private static Map<String, QueueMetrics> queueMetrics =
+ new HashMap<String, QueueMetrics>();
+
+ public synchronized
+ static QueueMetrics forQueue(MetricsSystem ms, String queueName,
Queue parent, boolean enableUserMetrics,
Configuration conf) {
- QueueMetrics metrics =
- new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf
- ).tag(QUEUE_INFO, queueName);
- return ms == null ? metrics : ms.register(sourceName(queueName).toString(),
- "Metrics for queue: " + queueName, metrics);
+ QueueMetrics metrics = queueMetrics.get(queueName);
+ if (metrics == null) {
+ metrics =
+ new QueueMetrics(ms, queueName, parent, enableUserMetrics, conf).
+ tag(QUEUE_INFO, queueName);
+
+ // Register with the MetricsSystems
+ if (ms != null) {
+ metrics =
+ ms.register(
+ sourceName(queueName).toString(),
+ "Metrics for queue: " + queueName, metrics);
+ }
+ queueMetrics.put(queueName, metrics);
+ }
+
+ return metrics;
}
public synchronized QueueMetrics getUserMetrics(String userName) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1375066&r1=1375065&r2=1375066&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Mon Aug 20 15:36:51 2012
@@ -38,14 +38,22 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
public class TestQueueMetrics {
static final int GB = 1024; // MB
private static final Configuration conf = new Configuration();
- final MetricsSystem ms = new MetricsSystemImpl();
+ private MetricsSystem ms;
+ @Before
+ public void setUp() {
+ ms = new MetricsSystemImpl();
+ QueueMetrics.clearQueueMetrics();
+ }
+
@Test public void testDefaultSingleQueueMetrics() {
String queueName = "single";
String user = "alice";
@@ -226,6 +234,37 @@ public class TestQueueMetrics {
checkApps(userSource, 1, 0, 0, 1, 0, 0);
checkApps(parentUserSource, 1, 0, 0, 1, 0, 0);
}
+
+ @Test
+ public void testMetricsCache() {
+ MetricsSystem ms = new MetricsSystemImpl("cache");
+
+ try {
+ String p1 = "root1";
+ String leafQueueName = "root1.leaf";
+ ms.start();
+
+ QueueMetrics p1Metrics =
+ QueueMetrics.forQueue(ms, p1, null, true, conf);
+ Queue parentQueue1 = make(stub(Queue.class).returning(p1Metrics).
+ from.getMetrics());
+ QueueMetrics metrics =
+ QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf);
+
+ Assert.assertNotNull("QueueMetrics for A shoudn't be null", metrics);
+
+ // Re-register to check for cache hit, shouldn't blow up metrics-system...
+ // also, verify parent-metrics
+ QueueMetrics alterMetrics =
+ QueueMetrics.forQueue(ms, leafQueueName, parentQueue1, true, conf);
+
+ Assert.assertNotNull("QueueMetrics for alterMetrics shoudn't be null",
+ alterMetrics);
+ } finally {
+ ms.shutdown();
+ }
+ }
+
public static void checkApps(MetricsSource source, int submitted, int pending,
int running, int completed, int failed, int killed) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1375066&r1=1375065&r2=1375066&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Mon Aug 20 15:36:51 2012
@@ -98,7 +98,8 @@ public class TestLeafQueue {
csConf =
new CapacitySchedulerConfiguration();
csConf.setBoolean("yarn.scheduler.capacity.user-metrics.enable", true);
- setupQueueConfiguration(csConf);
+ final String newRoot = "root" + System.currentTimeMillis();
+ setupQueueConfiguration(csConf, newRoot);
YarnConfiguration conf = new YarnConfiguration();
cs.setConf(conf);
@@ -112,7 +113,8 @@ public class TestLeafQueue {
when(csContext.getClusterResources()).
thenReturn(Resources.createResource(100 * 16 * GB));
root =
- CapacityScheduler.parseQueue(csContext, csConf, null, "root",
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT,
queues, queues,
CapacityScheduler.queueComparator,
CapacityScheduler.applicationComparator,
@@ -126,25 +128,33 @@ public class TestLeafQueue {
private static final String C = "c";
private static final String C1 = "c1";
private static final String D = "d";
- private void setupQueueConfiguration(CapacitySchedulerConfiguration conf) {
+ private void setupQueueConfiguration(
+ CapacitySchedulerConfiguration conf,
+ final String newRoot) {
// Define top-level queues
- conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {A, B, C, D});
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {newRoot});
conf.setCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setMaximumCapacity(CapacitySchedulerConfiguration.ROOT, 100);
conf.setAcl(CapacitySchedulerConfiguration.ROOT, QueueACL.SUBMIT_APPLICATIONS, " ");
- final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+ final String Q_newRoot = CapacitySchedulerConfiguration.ROOT + "." + newRoot;
+ conf.setQueues(Q_newRoot, new String[] {A, B, C, D});
+ conf.setCapacity(Q_newRoot, 100);
+ conf.setMaximumCapacity(Q_newRoot, 100);
+ conf.setAcl(Q_newRoot, QueueACL.SUBMIT_APPLICATIONS, " ");
+
+ final String Q_A = Q_newRoot + "." + A;
conf.setCapacity(Q_A, 8.5f);
conf.setMaximumCapacity(Q_A, 20);
conf.setAcl(Q_A, QueueACL.SUBMIT_APPLICATIONS, "*");
- final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+ final String Q_B = Q_newRoot + "." + B;
conf.setCapacity(Q_B, 80);
conf.setMaximumCapacity(Q_B, 99);
conf.setAcl(Q_B, QueueACL.SUBMIT_APPLICATIONS, "*");
- final String Q_C = CapacitySchedulerConfiguration.ROOT + "." + C;
+ final String Q_C = Q_newRoot + "." + C;
conf.setCapacity(Q_C, 1.5f);
conf.setMaximumCapacity(Q_C, 10);
conf.setAcl(Q_C, QueueACL.SUBMIT_APPLICATIONS, " ");
@@ -154,7 +164,7 @@ public class TestLeafQueue {
final String Q_C1 = Q_C + "." + C1;
conf.setCapacity(Q_C1, 100);
- final String Q_D = CapacitySchedulerConfiguration.ROOT + "." + D;
+ final String Q_D = Q_newRoot + "." + D;
conf.setCapacity(Q_D, 10);
conf.setMaximumCapacity(Q_D, 11);
conf.setAcl(Q_D, QueueACL.SUBMIT_APPLICATIONS, "user_d");