You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by su...@apache.org on 2012/11/08 20:09:59 UTC
svn commit: r1407217 [2/2] - in
/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project: ./
hadoop-yarn/conf/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-app...
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java Thu Nov 8 19:09:46 2012
@@ -69,7 +69,7 @@ public class RmView extends TwoColumnLay
append("null,{sType:'title-numeric', bSearchable:false}, null]");
// Sort by id upon page load
- init.append(", aaSorting: [[0, 'asc']]");
+ init.append(", aaSorting: [[0, 'desc']]");
String rows = $("rowlimit");
int rowLimit = rows.isEmpty() ? MAX_DISPLAY_ROWS : Integer.parseInt(rows);
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Nov 8 19:09:46 2012
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
+import static org.apache.hadoop.yarn.util.StringHelper.pjoin;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
@@ -85,6 +87,8 @@ public class TestRMAppAttemptTransitions
LogFactory.getLog(TestRMAppAttemptTransitions.class);
private static final String EMPTY_DIAGNOSTICS = "";
+ private static final String RM_WEBAPP_ADDR =
+ YarnConfiguration.getRMWebAppHostAndPort(new Configuration());
private RMContext rmContext;
private YarnScheduler scheduler;
@@ -203,7 +207,7 @@ public class TestRMAppAttemptTransitions
application = mock(RMApp.class);
applicationAttempt =
new RMAppAttemptImpl(applicationAttemptId, null, rmContext, scheduler,
- masterService, submissionContext, null);
+ masterService, submissionContext, new Configuration());
when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt);
when(application.getApplicationId()).thenReturn(applicationId);
@@ -216,6 +220,11 @@ public class TestRMAppAttemptTransitions
}
+ private String getProxyUrl(RMAppAttempt appAttempt) {
+ return pjoin(RM_WEBAPP_ADDR, "proxy",
+ appAttempt.getAppAttemptId().getApplicationId(), "");
+ }
+
/**
* {@link RMAppAttemptState#NEW}
*/
@@ -373,8 +382,8 @@ public class TestRMAppAttemptTransitions
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
- getApplicationId()+"/", applicationAttempt.getTrackingUrl());
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
// TODO - need to add more checks relevant to this state
}
@@ -390,8 +399,8 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
- getApplicationId()+"/", applicationAttempt.getTrackingUrl());
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(finalStatus, applicationAttempt.getFinalApplicationStatus());
}
@@ -408,8 +417,8 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
- assertEquals("null/proxy/"+applicationAttempt.getAppAttemptId().
- getApplicationId()+"/", applicationAttempt.getTrackingUrl());
+ assertEquals(getProxyUrl(applicationAttempt),
+ applicationAttempt.getTrackingUrl());
assertEquals(finishedContainerCount, applicationAttempt
.getJustFinishedContainers().size());
assertEquals(container, applicationAttempt.getMasterContainer());
@@ -596,7 +605,30 @@ public class TestRMAppAttemptTransitions
diagnostics));
testAppAttemptFailedState(amContainer, diagnostics);
}
-
+
+ @Test
+ public void testRunningToFailed() {
+ Container amContainer = allocateApplicationAttempt();
+ launchApplicationAttempt(amContainer);
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl");
+ String containerDiagMsg = "some error";
+ int exitCode = 123;
+ ContainerStatus cs = BuilderUtils.newContainerStatus(amContainer.getId(),
+ ContainerState.COMPLETE, containerDiagMsg, exitCode);
+ ApplicationAttemptId appAttemptId = applicationAttempt.getAppAttemptId();
+ applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
+ appAttemptId, cs));
+ assertEquals(RMAppAttemptState.FAILED,
+ applicationAttempt.getAppAttemptState());
+ assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
+ assertEquals(amContainer, applicationAttempt.getMasterContainer());
+ assertEquals(0, applicationAttempt.getRanNodes().size());
+ String rmAppPageUrl = pjoin(RM_WEBAPP_ADDR, "cluster", "app",
+ applicationAttempt.getAppAttemptId().getApplicationId());
+ assertEquals(rmAppPageUrl, applicationAttempt.getOriginalTrackingUrl());
+ assertEquals(rmAppPageUrl, applicationAttempt.getTrackingUrl());
+ }
+
@Test
public void testUnregisterToKilledFinishing() {
Container amContainer = allocateApplicationAttempt();
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Thu Nov 8 19:09:46 2012
@@ -66,8 +66,8 @@ public class TestCapacityScheduler {
private static float B_CAPACITY = 89.5f;
private static float A1_CAPACITY = 30;
private static float A2_CAPACITY = 70;
- private static float B1_CAPACITY = 50;
- private static float B2_CAPACITY = 30;
+ private static float B1_CAPACITY = 79.2f;
+ private static float B2_CAPACITY = 0.8f;
private static float B3_CAPACITY = 20;
private ResourceManager resourceManager = null;
@@ -378,4 +378,43 @@ public class TestCapacityScheduler {
Assert.assertEquals(4 * GB, cs.getClusterResources().getMemory());
}
+
+ @Test
+ public void testRefreshQueuesWithNewQueue() throws Exception {
+ CapacityScheduler cs = new CapacityScheduler();
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ cs.setConf(new YarnConfiguration());
+ cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new ClientToAMTokenSecretManagerInRM()));
+ checkQueueCapacities(cs, A_CAPACITY, B_CAPACITY);
+
+ // Add a new queue b4
+ String B4 = B + ".b4";
+ float B4_CAPACITY = 10;
+
+ B3_CAPACITY -= B4_CAPACITY;
+ try {
+ conf.setCapacity(A, 80f);
+ conf.setCapacity(B, 20f);
+ conf.setQueues(B, new String[] {"b1", "b2", "b3", "b4"});
+ conf.setCapacity(B1, B1_CAPACITY);
+ conf.setCapacity(B2, B2_CAPACITY);
+ conf.setCapacity(B3, B3_CAPACITY);
+ conf.setCapacity(B4, B4_CAPACITY);
+ cs.reinitialize(conf,null);
+ checkQueueCapacities(cs, 80f, 20f);
+
+ // Verify parent for B4
+ CSQueue rootQueue = cs.getRootQueue();
+ CSQueue queueB = findQueue(rootQueue, B);
+ CSQueue queueB4 = findQueue(queueB, B4);
+
+ assertEquals(queueB, queueB4.getParent());
+ } finally {
+ B3_CAPACITY += B4_CAPACITY;
+ }
+ }
+
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Thu Nov 8 19:09:46 2012
@@ -1259,7 +1259,7 @@ public class TestLeafQueue {
assignment = a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
any(Priority.class), any(ResourceRequest.class), any(Container.class));
- assertEquals(0, app_0.getSchedulingOpportunities(priority)); // should reset
+ assertEquals(4, app_0.getSchedulingOpportunities(priority)); // should NOT reset
assertEquals(2, app_0.getTotalRequiredResources(priority));
assertEquals(NodeType.OFF_SWITCH, assignment.getType());
@@ -1408,11 +1408,11 @@ public class TestLeafQueue {
assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
assertEquals(1, app_0.getTotalRequiredResources(priority_2));
- // Another off-switch, shouldn allocate OFF_SWITCH P1
+ // Another off-switch, shouldn't allocate OFF_SWITCH P1
a.assignContainers(clusterResource, node_2);
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_2),
eq(priority_1), any(ResourceRequest.class), any(Container.class));
- assertEquals(0, app_0.getSchedulingOpportunities(priority_1));
+ assertEquals(3, app_0.getSchedulingOpportunities(priority_1));
assertEquals(1, app_0.getTotalRequiredResources(priority_1));
verify(app_0, never()).allocate(any(NodeType.class), eq(node_2),
eq(priority_2), any(ResourceRequest.class), any(Container.class));
@@ -1438,7 +1438,7 @@ public class TestLeafQueue {
assertEquals(0, app_0.getTotalRequiredResources(priority_1));
verify(app_0).allocate(eq(NodeType.OFF_SWITCH), eq(node_1),
eq(priority_2), any(ResourceRequest.class), any(Container.class));
- assertEquals(0, app_0.getSchedulingOpportunities(priority_2));
+ assertEquals(1, app_0.getSchedulingOpportunities(priority_2));
assertEquals(0, app_0.getTotalRequiredResources(priority_2));
}
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java Thu Nov 8 19:09:46 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.re
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.doAnswer;
@@ -494,6 +495,72 @@ public class TestParentQueue {
verifyQueueMetrics(c, 4*GB, clusterResource);
reset(a); reset(b); reset(c);
}
+
+ @Test (expected=IllegalArgumentException.class)
+ public void testQueueCapacitySettingChildZero() throws Exception {
+ // Setup queue configs
+ setupMultiLevelQueues(csConf);
+
+ // set child queues capacity to 0 when parents not 0
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+ csConf.setCapacity(Q_B + "." + B1, 0);
+ csConf.setCapacity(Q_B + "." + B2, 0);
+ csConf.setCapacity(Q_B + "." + B3, 0);
+
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+ }
+
+ @Test (expected=IllegalArgumentException.class)
+ public void testQueueCapacitySettingParentZero() throws Exception {
+ // Setup queue configs
+ setupMultiLevelQueues(csConf);
+
+ // set parent capacity to 0 when child not 0
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+ csConf.setCapacity(Q_B, 0);
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+ csConf.setCapacity(Q_A, 60);
+
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+ }
+
+ @Test
+ public void testQueueCapacityZero() throws Exception {
+ // Setup queue configs
+ setupMultiLevelQueues(csConf);
+
+ // set parent and child capacity to 0
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+ csConf.setCapacity(Q_B, 0);
+ csConf.setCapacity(Q_B + "." + B1, 0);
+ csConf.setCapacity(Q_B + "." + B2, 0);
+ csConf.setCapacity(Q_B + "." + B3, 0);
+
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+ csConf.setCapacity(Q_A, 60);
+
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ try {
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues,
+ CapacityScheduler.queueComparator,
+ CapacityScheduler.applicationComparator,
+ TestUtils.spyHook);
+ } catch (IllegalArgumentException e) {
+ fail("Failed to create queues with 0 capacity: " + e);
+ }
+ assertTrue("Failed to create queues with 0 capacity", true);
+ }
@Test
public void testOffSwitchScheduling() throws Exception {
Modified: hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java?rev=1407217&r1=1407216&r2=1407217&view=diff
==============================================================================
--- hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java (original)
+++ hadoop/common/branches/branch-trunk-win/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java Thu Nov 8 19:09:46 2012
@@ -144,11 +144,11 @@ public class TestRMWebServicesCapacitySc
final String B2 = B + ".b2";
final String B3 = B + ".b3";
conf.setQueues(B, new String[] { "b1", "b2", "b3" });
- conf.setCapacity(B1, 50);
+ conf.setCapacity(B1, 60);
conf.setUserLimitFactor(B1, 100.0f);
- conf.setCapacity(B2, 30);
+ conf.setCapacity(B2, 39.5f);
conf.setUserLimitFactor(B2, 100.0f);
- conf.setCapacity(B3, 20);
+ conf.setCapacity(B3, 0.5f);
conf.setUserLimitFactor(B3, 100.0f);
conf.setQueues(A1, new String[] {"a1a", "a1b"});