You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/01/14 01:01:38 UTC
svn commit: r1557890 [4/4] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/mai...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Tue Jan 14 00:01:35 2014
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -66,8 +66,10 @@ public class TestQueueMetrics {
MetricsSource queueSource= queueSource(ms, queueName);
AppSchedulingInfo app = mockApp(user);
- metrics.submitApp(user, 1);
+ metrics.submitApp(user);
MetricsSource userSource = userSource(ms, queueName, user);
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ metrics.submitAppAttempt(user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100));
@@ -76,7 +78,7 @@ public class TestQueueMetrics {
// configurable cluster/queue resources
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
- metrics.incrAppsRunning(app, user);
+ metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2));
@@ -85,7 +87,10 @@ public class TestQueueMetrics {
metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2));
checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
- metrics.finishApp(app, RMAppAttemptState.FINISHED);
+ metrics.finishAppAttempt(
+ app.getApplicationId(), app.isPending(), app.getUser());
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ metrics.finishApp(user, RMAppState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
assertNull(userSource);
}
@@ -100,39 +105,47 @@ public class TestQueueMetrics {
MetricsSource queueSource = queueSource(ms, queueName);
AppSchedulingInfo app = mockApp(user);
- metrics.submitApp(user, 1);
+ metrics.submitApp(user);
MetricsSource userSource = userSource(ms, queueName, user);
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ metrics.submitAppAttempt(user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
- metrics.incrAppsRunning(app, user);
+ metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
- metrics.finishApp(app, RMAppAttemptState.FAILED);
- checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
+ metrics.finishAppAttempt(
+ app.getApplicationId(), app.isPending(), app.getUser());
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
- metrics.submitApp(user, 2);
+ metrics.submitAppAttempt(user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
- metrics.incrAppsRunning(app, user);
+ metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
// Suppose say application has failed this time as well.
- metrics.finishApp(app, RMAppAttemptState.FAILED);
- checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
+ metrics.finishAppAttempt(
+ app.getApplicationId(), app.isPending(), app.getUser());
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
// As the application has failed, framework retries the same application
// based on configuration
- metrics.submitApp(user, 3);
+ metrics.submitAppAttempt(user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
- metrics.incrAppsRunning(app, user);
+ metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
- // Suppose say application has finished.
- metrics.finishApp(app, RMAppAttemptState.FINISHED);
- checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
+ // Suppose say application has failed, and there's no more retries.
+ metrics.finishAppAttempt(
+ app.getApplicationId(), app.isPending(), app.getUser());
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+
+ metrics.finishApp(user, RMAppState.FAILED);
+ checkApps(queueSource, 1, 0, 0, 0, 1, 0, true);
assertNull(userSource);
}
@@ -146,9 +159,13 @@ public class TestQueueMetrics {
MetricsSource queueSource = queueSource(ms, queueName);
AppSchedulingInfo app = mockApp(user);
- metrics.submitApp(user, 1);
+ metrics.submitApp(user);
MetricsSource userSource = userSource(ms, queueName, user);
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
+
+ metrics.submitAppAttempt(user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
@@ -160,7 +177,7 @@ public class TestQueueMetrics {
checkResources(queueSource, 0, 0, 0, 0, 0, 100*GB, 100, 15*GB, 15, 5, 0, 0, 0);
checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
- metrics.incrAppsRunning(app, user);
+ metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
@@ -172,7 +189,11 @@ public class TestQueueMetrics {
checkResources(queueSource, 4*GB, 4, 2, 3, 1, 100*GB, 100, 9*GB, 9, 2, 0, 0, 0);
checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
- metrics.finishApp(app, RMAppAttemptState.FINISHED);
+ metrics.finishAppAttempt(
+ app.getApplicationId(), app.isPending(), app.getUser());
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
+ metrics.finishApp(user, RMAppState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
}
@@ -192,10 +213,16 @@ public class TestQueueMetrics {
MetricsSource queueSource = queueSource(ms, leafQueueName);
AppSchedulingInfo app = mockApp(user);
- metrics.submitApp(user, 1);
+ metrics.submitApp(user);
MetricsSource userSource = userSource(ms, leafQueueName, user);
MetricsSource parentUserSource = userSource(ms, parentQueueName, user);
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
+
+ metrics.submitAppAttempt(user);
checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);
checkApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true);
checkApps(userSource, 1, 1, 0, 0, 0, 0, true);
@@ -211,7 +238,7 @@ public class TestQueueMetrics {
checkResources(userSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
checkResources(parentUserSource, 0, 0, 0, 0, 0, 10*GB, 10, 15*GB, 15, 5, 0, 0, 0);
- metrics.incrAppsRunning(app, user);
+ metrics.runAppAttempt(app.getApplicationId(), user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);
checkApps(userSource, 1, 0, 1, 0, 0, 0, true);
@@ -231,7 +258,14 @@ public class TestQueueMetrics {
checkResources(userSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
checkResources(parentUserSource, 4*GB, 4, 2, 3, 1, 10*GB, 10, 9*GB, 9, 2, 0, 0, 0);
- metrics.finishApp(app, RMAppAttemptState.FINISHED);
+ metrics.finishAppAttempt(
+ app.getApplicationId(), app.isPending(), app.getUser());
+ checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(userSource, 1, 0, 0, 0, 0, 0, true);
+ checkApps(parentUserSource, 1, 0, 0, 0, 0, 0, true);
+
+ metrics.finishApp(user, RMAppState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
checkApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true);
checkApps(userSource, 1, 0, 0, 1, 0, 0, true);
@@ -308,7 +342,7 @@ public class TestQueueMetrics {
assertGauge("AppsPending", pending, rb);
assertGauge("AppsRunning", running, rb);
assertCounter("AppsCompleted", completed, rb);
- assertGauge("AppsFailed", failed, rb);
+ assertCounter("AppsFailed", failed, rb);
assertCounter("AppsKilled", killed, rb);
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Tue Jan 14 00:01:35 2014
@@ -562,18 +562,18 @@ public class TestCapacityScheduler {
new AppAddedSchedulerEvent(appId, "default", "user");
cs.handle(addAppEvent);
SchedulerEvent addAttemptEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
cs.handle(addAttemptEvent);
// Verify the blacklist can be updated independent of requesting containers
cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(),
Collections.singletonList(host), null);
- Assert.assertTrue(cs.getApplication(appAttemptId).isBlacklisted(host));
+ Assert.assertTrue(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
Collections.singletonList(host));
- Assert.assertFalse(cs.getApplication(appAttemptId).isBlacklisted(host));
+ Assert.assertFalse(cs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
rm.stop();
}
@@ -597,66 +597,6 @@ public class TestCapacityScheduler {
assertTrue(appComparator.compare(app1, app3) < 0);
assertTrue(appComparator.compare(app2, app3) < 0);
}
-
- @Test
- public void testConcurrentAccessOnApplications() throws Exception {
- CapacityScheduler cs = new CapacityScheduler();
- verifyConcurrentAccessOnApplications(
- cs.appAttempts, FiCaSchedulerApp.class, Queue.class);
- }
-
- public static <T extends SchedulerApplicationAttempt, Q extends Queue>
- void verifyConcurrentAccessOnApplications(
- final Map<ApplicationAttemptId, T> applications, Class<T> appClazz,
- final Class<Q> queueClazz)
- throws Exception {
- final int size = 10000;
- final ApplicationId appId = ApplicationId.newInstance(0, 0);
- final Constructor<T> ctor = appClazz.getDeclaredConstructor(
- ApplicationAttemptId.class, String.class, queueClazz,
- ActiveUsersManager.class, RMContext.class);
-
- ApplicationAttemptId appAttemptId0
- = ApplicationAttemptId.newInstance(appId, 0);
- applications.put(appAttemptId0, ctor.newInstance(
- appAttemptId0, null, mock(queueClazz), null, null));
- assertNotNull(applications.get(appAttemptId0));
-
- // Imitating the thread of scheduler that will add and remove apps
- final AtomicBoolean finished = new AtomicBoolean(false);
- final AtomicBoolean failed = new AtomicBoolean(false);
- Thread t = new Thread() {
-
- @Override
- public void run() {
- for (int i = 1; i <= size; ++i) {
- ApplicationAttemptId appAttemptId
- = ApplicationAttemptId.newInstance(appId, i);
- try {
- applications.put(appAttemptId, ctor.newInstance(
- appAttemptId, null, mock(queueClazz), null, null));
- } catch (Exception e) {
- failed.set(true);
- finished.set(true);
- return;
- }
- }
- for (int i = 1; i <= size; ++i) {
- ApplicationAttemptId appAttemptId
- = ApplicationAttemptId.newInstance(appId, i);
- applications.remove(appAttemptId);
- }
- finished.set(true);
- }
- };
- t.start();
-
- // Imitating the thread of rmappattempt that will get the app
- while (!finished.get()) {
- assertNotNull(applications.get(appAttemptId0));
- }
- assertFalse(failed.get());
- }
@Test
public void testGetAppsInQueue() throws Exception {
@@ -702,7 +642,7 @@ public class TestCapacityScheduler {
SchedulerApplication app =
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
- cs.applications, cs, "a1");
+ cs.getSchedulerApplications(), cs, "a1");
Assert.assertEquals("a1", app.getQueue().getQueueName());
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Jan 14 00:01:35 2014
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -63,7 +64,10 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -345,15 +349,20 @@ public class TestLeafQueue {
.getMockApplicationAttemptId(0, 1);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null,
rmContext);
- a.submitApplicationAttempt(app_0, user_0);
-
- when(cs.getApplication(appAttemptId_0)).thenReturn(app_0);
+ AppAddedSchedulerEvent addAppEvent =
+ new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(),
+ a.getQueueName(), user_0);
+ cs.handle(addAppEvent);
+ AppAttemptAddedSchedulerEvent addAttemptEvent =
+ new AppAttemptAddedSchedulerEvent(appAttemptId_0, false);
+ cs.handle(addAttemptEvent);
+
AppAttemptRemovedSchedulerEvent event = new AppAttemptRemovedSchedulerEvent(
- appAttemptId_0, RMAppAttemptState.FAILED);
+ appAttemptId_0, RMAppAttemptState.FAILED, false);
cs.handle(event);
assertEquals(0, a.getMetrics().getAppsPending());
- assertEquals(1, a.getMetrics().getAppsFailed());
+ assertEquals(0, a.getMetrics().getAppsFailed());
// Attempt the same application again
final ApplicationAttemptId appAttemptId_1 = TestUtils
@@ -365,10 +374,12 @@ public class TestLeafQueue {
assertEquals(1, a.getMetrics().getAppsSubmitted());
assertEquals(1, a.getMetrics().getAppsPending());
- when(cs.getApplication(appAttemptId_1)).thenReturn(app_0);
event = new AppAttemptRemovedSchedulerEvent(appAttemptId_0,
- RMAppAttemptState.FINISHED);
+ RMAppAttemptState.FINISHED, false);
cs.handle(event);
+ AppRemovedSchedulerEvent rEvent = new AppRemovedSchedulerEvent(
+ appAttemptId_0.getApplicationId(), RMAppState.FINISHED);
+ cs.handle(rEvent);
assertEquals(1, a.getMetrics().getAppsSubmitted());
assertEquals(0, a.getMetrics().getAppsPending());
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Tue Jan 14 00:01:35 2014
@@ -79,7 +79,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
@@ -260,8 +259,8 @@ public class TestFairScheduler {
scheduler.addApplication(id.getApplicationId(), queueId, userId);
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
- if (scheduler.applications.containsKey(id.getApplicationId())) {
- scheduler.addApplicationAttempt(id);
+ if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
+ scheduler.addApplicationAttempt(id, false);
}
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY,
@@ -590,7 +589,7 @@ public class TestFairScheduler {
// Make sure queue 2 is waiting with a reservation
assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
getResourceUsage().getMemory());
- assertEquals(1024, scheduler.appAttempts.get(attId).getCurrentReservation().getMemory());
+ assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemory());
// Now another node checks in with capacity
RMNode node2 =
@@ -606,10 +605,10 @@ public class TestFairScheduler {
getResourceUsage().getMemory());
// The old reservation should still be there...
- assertEquals(1024, scheduler.appAttempts.get(attId).getCurrentReservation().getMemory());
+ assertEquals(1024, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemory());
// ... but it should disappear when we update the first node.
scheduler.handle(updateEvent);
- assertEquals(0, scheduler.appAttempts.get(attId).getCurrentReservation().getMemory());
+ assertEquals(0, scheduler.getSchedulerApp(attId).getCurrentReservation().getMemory());
}
@@ -630,7 +629,7 @@ public class TestFairScheduler {
"user1");
scheduler.handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attempAddedEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
scheduler.handle(attempAddedEvent);
assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
.getRunnableAppSchedulables().size());
@@ -656,7 +655,7 @@ public class TestFairScheduler {
"user2");
scheduler.handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attempAddedEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
scheduler.handle(attempAddedEvent);
assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
.getRunnableAppSchedulables().size());
@@ -710,7 +709,6 @@ public class TestFairScheduler {
scheduler.reinitialize(conf, resourceManager.getRMContext());
ApplicationAttemptId appId;
- Map<ApplicationAttemptId, FSSchedulerApp> apps = scheduler.appAttempts;
List<QueuePlacementRule> rules = new ArrayList<QueuePlacementRule>();
rules.add(new QueuePlacementRule.Specified().initialize(true, null));
@@ -723,17 +721,17 @@ public class TestFairScheduler {
scheduler.getAllocationConfiguration().placementPolicy =
new QueuePlacementPolicy(rules, queues, conf);
appId = createSchedulingRequest(1024, "somequeue", "user1");
- assertEquals("root.somequeue", apps.get(appId).getQueueName());
+ assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "user1");
- assertEquals("root.user1", apps.get(appId).getQueueName());
+ assertEquals("root.user1", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "user3");
- assertEquals("root.user3group", apps.get(appId).getQueueName());
+ assertEquals("root.user3group", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "user4");
- assertEquals("root.user4subgroup1", apps.get(appId).getQueueName());
+ assertEquals("root.user4subgroup1", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "user5");
- assertEquals("root.user5subgroup2", apps.get(appId).getQueueName());
+ assertEquals("root.user5subgroup2", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "otheruser");
- assertEquals("root.default", apps.get(appId).getQueueName());
+ assertEquals("root.default", scheduler.getSchedulerApp(appId).getQueueName());
// test without specified as first rule
rules = new ArrayList<QueuePlacementRule>();
@@ -743,11 +741,11 @@ public class TestFairScheduler {
scheduler.getAllocationConfiguration().placementPolicy =
new QueuePlacementPolicy(rules, queues, conf);
appId = createSchedulingRequest(1024, "somequeue", "user1");
- assertEquals("root.user1", apps.get(appId).getQueueName());
+ assertEquals("root.user1", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "somequeue", "otheruser");
- assertEquals("root.somequeue", apps.get(appId).getQueueName());
+ assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "otheruser");
- assertEquals("root.default", apps.get(appId).getQueueName());
+ assertEquals("root.default", scheduler.getSchedulerApp(appId).getQueueName());
}
@Test
@@ -802,13 +800,13 @@ public class TestFairScheduler {
ApplicationAttemptId id11 = createAppAttemptId(1, 1);
scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1");
- scheduler.addApplicationAttempt(id11);
+ scheduler.addApplicationAttempt(id11, false);
ApplicationAttemptId id21 = createAppAttemptId(2, 1);
scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1");
- scheduler.addApplicationAttempt(id21);
+ scheduler.addApplicationAttempt(id21, false);
ApplicationAttemptId id22 = createAppAttemptId(2, 2);
scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1");
- scheduler.addApplicationAttempt(id22);
+ scheduler.addApplicationAttempt(id22, false);
int minReqSize =
FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB;
@@ -854,7 +852,7 @@ public class TestFairScheduler {
"user1");
scheduler.handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attemptAddedEvent =
- new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1));
+ new AppAttemptAddedSchedulerEvent(createAppAttemptId(1, 1), false);
scheduler.handle(attemptAddedEvent);
// Scheduler should have two queues (the default and the one created for user1)
@@ -865,7 +863,7 @@ public class TestFairScheduler {
.getRunnableAppSchedulables().size());
AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent(
- createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);
+ createAppAttemptId(1, 1), RMAppAttemptState.FINISHED, false);
// Now remove app
scheduler.handle(appRemovedEvent1);
@@ -1138,12 +1136,12 @@ public class TestFairScheduler {
scheduler.handle(nodeUpdate3);
}
- assertEquals(1, scheduler.appAttempts.get(app1).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app2).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app3).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app4).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app5).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app6).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app5).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app6).getLiveContainers().size());
// Now new requests arrive from queues C and D
ApplicationAttemptId app7 =
@@ -1166,16 +1164,16 @@ public class TestFairScheduler {
// Make sure it is lowest priority container.
scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
Resources.createResource(2 * 1024));
- assertEquals(1, scheduler.appAttempts.get(app1).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app2).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app4).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app5).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app2).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app5).getLiveContainers().size());
// First verify we are adding containers to preemption list for the application
- assertTrue(!Collections.disjoint(scheduler.appAttempts.get(app3).getLiveContainers(),
- scheduler.appAttempts.get(app3).getPreemptionContainers()));
- assertTrue(!Collections.disjoint(scheduler.appAttempts.get(app6).getLiveContainers(),
- scheduler.appAttempts.get(app6).getPreemptionContainers()));
+ assertTrue(!Collections.disjoint(scheduler.getSchedulerApp(app3).getLiveContainers(),
+ scheduler.getSchedulerApp(app3).getPreemptionContainers()));
+ assertTrue(!Collections.disjoint(scheduler.getSchedulerApp(app6).getLiveContainers(),
+ scheduler.getSchedulerApp(app6).getPreemptionContainers()));
// Pretend 15 seconds have passed
clock.tick(15);
@@ -1185,8 +1183,8 @@ public class TestFairScheduler {
Resources.createResource(2 * 1024));
// At this point the containers should have been killed (since we are not simulating AM)
- assertEquals(0, scheduler.appAttempts.get(app6).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app3).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size());
// Trigger a kill by insisting we want containers back
scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
@@ -1200,22 +1198,22 @@ public class TestFairScheduler {
scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
Resources.createResource(2 * 1024));
- assertEquals(1, scheduler.appAttempts.get(app1).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app2).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app3).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app4).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app5).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app6).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app5).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size());
// Now A and B are below fair share, so preemption shouldn't do anything
scheduler.preemptResources(scheduler.getQueueManager().getLeafQueues(),
Resources.createResource(2 * 1024));
- assertEquals(1, scheduler.appAttempts.get(app1).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app2).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app3).getLiveContainers().size());
- assertEquals(1, scheduler.appAttempts.get(app4).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app5).getLiveContainers().size());
- assertEquals(0, scheduler.appAttempts.get(app6).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app1).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app2).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app3).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(app4).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app5).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(app6).getLiveContainers().size());
}
@Test (timeout = 5000)
@@ -1374,9 +1372,9 @@ public class TestFairScheduler {
// One container should get reservation and the other should get nothing
assertEquals(1024,
- scheduler.appAttempts.get(attId1).getCurrentReservation().getMemory());
+ scheduler.getSchedulerApp(attId1).getCurrentReservation().getMemory());
assertEquals(0,
- scheduler.appAttempts.get(attId2).getCurrentReservation().getMemory());
+ scheduler.getSchedulerApp(attId2).getCurrentReservation().getMemory());
}
@Test (timeout = 5000)
@@ -1411,7 +1409,7 @@ public class TestFairScheduler {
scheduler.handle(updateEvent);
// App 1 should be running
- assertEquals(1, scheduler.appAttempts.get(attId1).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
"user1", 1);
@@ -1420,7 +1418,7 @@ public class TestFairScheduler {
scheduler.handle(updateEvent);
// App 2 should not be running
- assertEquals(0, scheduler.appAttempts.get(attId2).getLiveContainers().size());
+ assertEquals(0, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
// Request another container for app 1
createSchedulingRequestExistingApplication(1024, 1, attId1);
@@ -1429,7 +1427,7 @@ public class TestFairScheduler {
scheduler.handle(updateEvent);
// Request should be fulfilled
- assertEquals(2, scheduler.appAttempts.get(attId1).getLiveContainers().size());
+ assertEquals(2, scheduler.getSchedulerApp(attId1).getLiveContainers().size());
}
@Test (timeout = 5000)
@@ -1449,10 +1447,10 @@ public class TestFairScheduler {
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(updateEvent);
- FSSchedulerApp app = scheduler.appAttempts.get(attId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
assertEquals(1, app.getLiveContainers().size());
- ContainerId containerId = scheduler.appAttempts.get(attId)
+ ContainerId containerId = scheduler.getSchedulerApp(attId)
.getLiveContainers().iterator().next().getContainerId();
// Cause reservation to be created
@@ -1521,9 +1519,9 @@ public class TestFairScheduler {
ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
"norealuserhasthisname2", 1);
- FSSchedulerApp app1 = scheduler.appAttempts.get(attId1);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1);
assertNotNull("The application was not allowed", app1);
- FSSchedulerApp app2 = scheduler.appAttempts.get(attId2);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2);
assertNull("The application was allowed", app2);
}
@@ -1547,7 +1545,7 @@ public class TestFairScheduler {
ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
scheduler.addApplication(appId.getApplicationId(), "queue1", "user1");
- scheduler.addApplicationAttempt(appId);
+ scheduler.addApplicationAttempt(appId, false);
// 1 request with 2 nodes on the same rack. another request with 1 node on
// a different rack
@@ -1566,14 +1564,14 @@ public class TestFairScheduler {
NodeUpdateSchedulerEvent updateEvent1 = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(updateEvent1);
// should assign node local
- assertEquals(1, scheduler.appAttempts.get(appId).getLiveContainers().size());
+ assertEquals(1, scheduler.getSchedulerApp(appId).getLiveContainers().size());
// node 2 checks in
scheduler.update();
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2);
scheduler.handle(updateEvent2);
// should assign rack local
- assertEquals(2, scheduler.appAttempts.get(appId).getLiveContainers().size());
+ assertEquals(2, scheduler.getSchedulerApp(appId).getLiveContainers().size());
}
@Test (timeout = 5000)
@@ -1592,8 +1590,8 @@ public class TestFairScheduler {
"user1", 2);
ApplicationAttemptId attId2 = createSchedulingRequest(1024, "queue1",
"user1", 2);
- FSSchedulerApp app1 = scheduler.appAttempts.get(attId1);
- FSSchedulerApp app2 = scheduler.appAttempts.get(attId2);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2);
FSLeafQueue queue1 = scheduler.getQueueManager().getLeafQueue("queue1", true);
queue1.setPolicy(new FifoPolicy());
@@ -1633,7 +1631,7 @@ public class TestFairScheduler {
ApplicationAttemptId attId =
createSchedulingRequest(1024, "root.default", "user", 8);
- FSSchedulerApp app = scheduler.appAttempts.get(attId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
// set maxAssign to 2: only 2 containers should be allocated
scheduler.maxAssign = 2;
@@ -1695,10 +1693,10 @@ public class TestFairScheduler {
ApplicationAttemptId attId4 =
createSchedulingRequest(1024, fifoQueue, user, 4);
- FSSchedulerApp app1 = scheduler.appAttempts.get(attId1);
- FSSchedulerApp app2 = scheduler.appAttempts.get(attId2);
- FSSchedulerApp app3 = scheduler.appAttempts.get(attId3);
- FSSchedulerApp app4 = scheduler.appAttempts.get(attId4);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2);
+ FSSchedulerApp app3 = scheduler.getSchedulerApp(attId3);
+ FSSchedulerApp app4 = scheduler.getSchedulerApp(attId4);
scheduler.getQueueManager().getLeafQueue(fifoQueue, true)
.setPolicy(SchedulingPolicy.parse("fifo"));
@@ -1813,7 +1811,7 @@ public class TestFairScheduler {
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.handle(updateEvent);
- FSSchedulerApp app = scheduler.appAttempts.get(attId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
assertEquals(0, app.getLiveContainers().size());
assertEquals(0, app.getReservedContainers().size());
@@ -1882,7 +1880,7 @@ public class TestFairScheduler {
NodeUpdateSchedulerEvent node2UpdateEvent = new NodeUpdateSchedulerEvent(node2);
// no matter how many heartbeats, node2 should never get a container
- FSSchedulerApp app = scheduler.appAttempts.get(attId1);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId1);
for (int i = 0; i < 10; i++) {
scheduler.handle(node2UpdateEvent);
assertEquals(0, app.getLiveContainers().size());
@@ -1921,7 +1919,7 @@ public class TestFairScheduler {
NodeUpdateSchedulerEvent node2UpdateEvent = new NodeUpdateSchedulerEvent(node2);
// no matter how many heartbeats, node2 should never get a container
- FSSchedulerApp app = scheduler.appAttempts.get(attId1);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId1);
for (int i = 0; i < 10; i++) {
scheduler.handle(node2UpdateEvent);
assertEquals(0, app.getLiveContainers().size());
@@ -1954,7 +1952,7 @@ public class TestFairScheduler {
ApplicationAttemptId attId = createSchedulingRequest(1024, "queue1",
"user1", 0);
- FSSchedulerApp app = scheduler.appAttempts.get(attId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
ResourceRequest nodeRequest = createResourceRequest(1024, node2.getHostName(), 1, 2, true);
ResourceRequest rackRequest = createResourceRequest(1024, "rack1", 1, 2, true);
@@ -1994,7 +1992,7 @@ public class TestFairScheduler {
ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default",
"user1", 2);
- FSSchedulerApp app = scheduler.appAttempts.get(attId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
scheduler.update();
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
@@ -2014,10 +2012,10 @@ public class TestFairScheduler {
ApplicationAttemptId appAttId1 = createSchedulingRequest(2048, 1, "queue1",
"user1", 2);
- FSSchedulerApp app1 = scheduler.appAttempts.get(appAttId1);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(appAttId1);
ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 2, "queue1",
"user1", 2);
- FSSchedulerApp app2 = scheduler.appAttempts.get(appAttId2);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
drfPolicy.initialize(scheduler.getClusterCapacity());
@@ -2055,13 +2053,13 @@ public class TestFairScheduler {
ApplicationAttemptId appAttId1 = createSchedulingRequest(3072, 1, "queue1",
"user1", 2);
- FSSchedulerApp app1 = scheduler.appAttempts.get(appAttId1);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(appAttId1);
ApplicationAttemptId appAttId2 = createSchedulingRequest(2048, 2, "queue1",
"user1", 2);
- FSSchedulerApp app2 = scheduler.appAttempts.get(appAttId2);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2);
ApplicationAttemptId appAttId3 = createSchedulingRequest(1024, 2, "queue2",
"user1", 2);
- FSSchedulerApp app3 = scheduler.appAttempts.get(appAttId3);
+ FSSchedulerApp app3 = scheduler.getSchedulerApp(appAttId3);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
drfPolicy.initialize(scheduler.getClusterCapacity());
@@ -2092,19 +2090,19 @@ public class TestFairScheduler {
ApplicationAttemptId appAttId1 = createSchedulingRequest(3074, 1, "queue1.subqueue1",
"user1", 2);
Thread.sleep(3); // so that start times will be different
- FSSchedulerApp app1 = scheduler.appAttempts.get(appAttId1);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(appAttId1);
ApplicationAttemptId appAttId2 = createSchedulingRequest(1024, 3, "queue1.subqueue1",
"user1", 2);
Thread.sleep(3); // so that start times will be different
- FSSchedulerApp app2 = scheduler.appAttempts.get(appAttId2);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(appAttId2);
ApplicationAttemptId appAttId3 = createSchedulingRequest(2048, 2, "queue1.subqueue2",
"user1", 2);
Thread.sleep(3); // so that start times will be different
- FSSchedulerApp app3 = scheduler.appAttempts.get(appAttId3);
+ FSSchedulerApp app3 = scheduler.getSchedulerApp(appAttId3);
ApplicationAttemptId appAttId4 = createSchedulingRequest(1024, 2, "queue2",
"user1", 2);
Thread.sleep(3); // so that start times will be different
- FSSchedulerApp app4 = scheduler.appAttempts.get(appAttId4);
+ FSSchedulerApp app4 = scheduler.getSchedulerApp(appAttId4);
DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
drfPolicy.initialize(scheduler.getClusterCapacity());
@@ -2184,7 +2182,7 @@ public class TestFairScheduler {
NodeUpdateSchedulerEvent(node2);
// no matter how many heartbeats, node2 should never get a container
- FSSchedulerApp app = scheduler.appAttempts.get(attId1);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId1);
for (int i = 0; i < 10; i++) {
scheduler.handle(node2UpdateEvent);
assertEquals(0, app.getLiveContainers().size());
@@ -2195,16 +2193,8 @@ public class TestFairScheduler {
assertEquals(1, app.getLiveContainers().size());
}
- @Test
- public void testConcurrentAccessOnApplications() throws Exception {
- FairScheduler fs = new FairScheduler();
- TestCapacityScheduler.verifyConcurrentAccessOnApplications(
- fs.appAttempts, FSSchedulerApp.class, FSLeafQueue.class);
- }
-
-
private void verifyAppRunnable(ApplicationAttemptId attId, boolean runnable) {
- FSSchedulerApp app = scheduler.appAttempts.get(attId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
FSLeafQueue queue = app.getQueue();
Collection<AppSchedulable> runnableApps =
queue.getRunnableAppSchedulables();
@@ -2260,7 +2250,7 @@ public class TestFairScheduler {
// Remove app 1 and both app 2 and app 4 should becomes runnable in its place
AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
- new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED);
+ new AppAttemptRemovedSchedulerEvent(attId1, RMAppAttemptState.FINISHED, false);
scheduler.handle(appRemovedEvent1);
verifyAppRunnable(attId2, true);
verifyQueueNumRunnable("queue2", 1, 0);
@@ -2324,7 +2314,7 @@ public class TestFairScheduler {
// Even though the app was removed from sub3, the app from sub2 gets to go
// because it came in first
AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
- new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED);
+ new AppAttemptRemovedSchedulerEvent(attId2, RMAppAttemptState.FINISHED, false);
scheduler.handle(appRemovedEvent1);
verifyAppRunnable(attId4, true);
verifyQueueNumRunnable("queue1.sub2", 2, 0);
@@ -2333,7 +2323,7 @@ public class TestFairScheduler {
// Now test removal of a non-runnable app
AppAttemptRemovedSchedulerEvent appRemovedEvent2 =
- new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED);
+ new AppAttemptRemovedSchedulerEvent(attId5, RMAppAttemptState.KILLED, true);
scheduler.handle(appRemovedEvent2);
assertEquals(0, scheduler.maxRunningEnforcer.usersNonRunnableApps
.get("user1").size());
@@ -2341,7 +2331,7 @@ public class TestFairScheduler {
verifyQueueNumRunnable("queue1.sub3", 0, 0);
// verify it doesn't become runnable when there would be space for it
AppAttemptRemovedSchedulerEvent appRemovedEvent3 =
- new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED);
+ new AppAttemptRemovedSchedulerEvent(attId4, RMAppAttemptState.FINISHED, true);
scheduler.handle(appRemovedEvent3);
verifyQueueNumRunnable("queue1.sub2", 1, 0);
verifyQueueNumRunnable("queue1.sub3", 0, 0);
@@ -2378,7 +2368,7 @@ public class TestFairScheduler {
ApplicationAttemptId appAttemptId =
createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
fs.addApplication(appAttemptId.getApplicationId(), "queue11", "user11");
- fs.addApplicationAttempt(appAttemptId);
+ fs.addApplicationAttempt(appAttemptId, false);
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ResourceRequest request =
createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
@@ -2389,7 +2379,7 @@ public class TestFairScheduler {
// at least one pass
Thread.sleep(fs.getConf().getContinuousSchedulingSleepMs() + 500);
- FSSchedulerApp app = fs.appAttempts.get(appAttemptId);
+ FSSchedulerApp app = fs.getSchedulerApp(appAttemptId);
// Wait until app gets resources.
while (app.getCurrentConsumption().equals(Resources.none())) { }
@@ -2477,7 +2467,7 @@ public class TestFairScheduler {
ApplicationAttemptId appAttemptId =
createSchedulingRequest(GB, "root.default", "user", 1);
- FSSchedulerApp app = scheduler.appAttempts.get(appAttemptId);
+ FSSchedulerApp app = scheduler.getSchedulerApp(appAttemptId);
// Verify the blacklist can be updated independent of requesting containers
scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
@@ -2487,7 +2477,7 @@ public class TestFairScheduler {
scheduler.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
Collections.singletonList(host));
- assertFalse(scheduler.appAttempts.get(appAttemptId).isBlacklisted(host));
+ assertFalse(scheduler.getSchedulerApp(appAttemptId).isBlacklisted(host));
List<ResourceRequest> update = Arrays.asList(
createResourceRequest(GB, node.getHostName(), 1, 0, true));
@@ -2555,6 +2545,6 @@ public class TestFairScheduler {
FairScheduler scheduler =
(FairScheduler) resourceManager.getResourceScheduler();
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
- scheduler.applications, scheduler, "default");
+ scheduler.getSchedulerApplications(), scheduler, "default");
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Tue Jan 14 00:01:35 2014
@@ -70,7 +70,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -156,17 +155,12 @@ public class TestFifoScheduler {
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
schedular.handle(appEvent);
SchedulerEvent attemptEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
schedular.handle(attemptEvent);
appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
-
- SchedulerEvent appEvent2 =
- new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue",
- "user");
- schedular.handle(appEvent2);
SchedulerEvent attemptEvent2 =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
schedular.handle(attemptEvent2);
int afterAppsSubmitted = metrics.getAppsSubmitted();
@@ -203,7 +197,7 @@ public class TestFifoScheduler {
"user1");
scheduler.handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
scheduler.handle(attemptEvent);
int memory = 64;
@@ -293,7 +287,7 @@ public class TestFifoScheduler {
"user1");
scheduler.handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
scheduler.handle(attemptEvent);
int memory = 1024;
@@ -534,13 +528,6 @@ public class TestFifoScheduler {
LOG.info("--- END: testFifoScheduler ---");
}
- @Test
- public void testConcurrentAccessOnApplications() throws Exception {
- FifoScheduler fs = new FifoScheduler();
- TestCapacityScheduler.verifyConcurrentAccessOnApplications(
- fs.appAttempts, FiCaSchedulerApp.class, Queue.class);
- }
-
@SuppressWarnings("resource")
@Test
public void testBlackListNodes() throws Exception {
@@ -564,18 +551,18 @@ public class TestFifoScheduler {
"user");
fs.handle(appEvent);
SchedulerEvent attemptEvent =
- new AppAttemptAddedSchedulerEvent(appAttemptId);
+ new AppAttemptAddedSchedulerEvent(appAttemptId, false);
fs.handle(attemptEvent);
// Verify the blacklist can be updated independent of requesting containers
fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(),
Collections.singletonList(host), null);
- Assert.assertTrue(fs.getApplication(appAttemptId).isBlacklisted(host));
+ Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
Collections.<ContainerId>emptyList(), null,
Collections.singletonList(host));
- Assert.assertFalse(fs.getApplication(appAttemptId).isBlacklisted(host));
+ Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
rm.stop();
}
@@ -604,8 +591,8 @@ public class TestFifoScheduler {
ResourceScheduler.class);
MockRM rm = new MockRM(conf);
FifoScheduler fs = (FifoScheduler)rm.getResourceScheduler();
- TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(fs.applications,
- fs, "queue");
+ TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
+ fs.getSchedulerApplications(), fs, "queue");
}
private void checkApplicationResourceUsage(int expected,
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Tue Jan 14 00:01:35 2014
@@ -29,6 +29,7 @@ import javax.xml.parsers.DocumentBuilder
import javax.xml.parsers.DocumentBuilderFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
@@ -267,6 +268,7 @@ public class TestRMWebServices extends J
verifyClusterGeneric(WebServicesTestUtils.getXmlLong(element, "id"),
WebServicesTestUtils.getXmlLong(element, "startedOn"),
WebServicesTestUtils.getXmlString(element, "state"),
+ WebServicesTestUtils.getXmlString(element, "haState"),
WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"),
WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"),
WebServicesTestUtils.getXmlString(element, "hadoopVersion"),
@@ -282,9 +284,10 @@ public class TestRMWebServices extends J
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("clusterInfo");
- assertEquals("incorrect number of elements", 9, info.length());
+ assertEquals("incorrect number of elements", 10, info.length());
verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"),
- info.getString("state"), info.getString("hadoopVersionBuiltOn"),
+ info.getString("state"), info.getString("haState"),
+ info.getString("hadoopVersionBuiltOn"),
info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"),
info.getString("resourceManagerVersionBuiltOn"),
info.getString("resourceManagerBuildVersion"),
@@ -293,9 +296,10 @@ public class TestRMWebServices extends J
}
public void verifyClusterGeneric(long clusterid, long startedon,
- String state, String hadoopVersionBuiltOn, String hadoopBuildVersion,
- String hadoopVersion, String resourceManagerVersionBuiltOn,
- String resourceManagerBuildVersion, String resourceManagerVersion) {
+ String state, String haState, String hadoopVersionBuiltOn,
+ String hadoopBuildVersion, String hadoopVersion,
+ String resourceManagerVersionBuiltOn, String resourceManagerBuildVersion,
+ String resourceManagerVersion) {
assertEquals("clusterId doesn't match: ",
ResourceManager.getClusterTimeStamp(), clusterid);
@@ -303,6 +307,8 @@ public class TestRMWebServices extends J
ResourceManager.getClusterTimeStamp(), startedon);
assertTrue("stated doesn't match: " + state,
state.matches(STATE.INITED.toString()));
+ assertTrue("HA state doesn't match: " + haState,
+ haState.matches("INITIALIZING"));
WebServicesTestUtils.checkStringMatch("hadoopVersionBuiltOn",
VersionInfo.getDate(), hadoopVersionBuiltOn);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Tue Jan 14 00:01:35 2014
@@ -41,9 +41,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -1387,29 +1384,30 @@ public class TestRMWebServicesApps exten
rm.stop();
}
- @Test
+ @Test (timeout = 20000)
public void testMultipleAppAttempts() throws JSONException, Exception {
rm.start();
- MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
- amNodeManager.nodeHeartbeat(true);
- rm.waitForState(app1.getCurrentAppAttempt().getAppAttemptId(),
- RMAppAttemptState.ALLOCATED);
+ MockAM am = MockRM.launchAM(app1, rm, amNodeManager);
int maxAppAttempts = rm.getConfig().getInt(
YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
assertTrue(maxAppAttempts > 1);
- int retriesLeft = maxAppAttempts;
- while (--retriesLeft > 0) {
- RMAppEvent event =
- new RMAppFailedAttemptEvent(app1.getApplicationId(),
- RMAppEventType.ATTEMPT_FAILED, "");
- app1.handle(event);
+ int numAttempt = 1;
+ while (true) {
+ // fail the AM by sending CONTAINER_FINISHED event without registering.
+ amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ am.waitForState(RMAppAttemptState.FAILED);
+ if (numAttempt == maxAppAttempts) {
+ rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
+ break;
+ }
+ // wait for app to start a new attempt.
rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
- amNodeManager.nodeHeartbeat(true);
+ am = MockRM.launchAM(app1, rm, amNodeManager);
+ numAttempt++;
}
- rm.waitForState(app1.getCurrentAppAttempt().getAppAttemptId(),
- RMAppAttemptState.ALLOCATED);
assertEquals("incorrect number of attempts", maxAppAttempts,
app1.getAppAttempts().values().size());
testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Tue Jan 14 00:01:35 2014
@@ -32,29 +32,101 @@
</properties>
<dependencies>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<scope>test</scope>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
</dependencies>
<build>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Tue Jan 14 00:01:35 2014
@@ -38,14 +38,94 @@
<artifactId>servlet-api</artifactId>
<scope>compile</scope>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1-jetty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</dependency>
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.glassfish.grizzly</groupId>
+ <artifactId>grizzly-http-servlet</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml Tue Jan 14 00:01:35 2014
@@ -32,14 +32,7 @@
<hadoop.common.build.dir>${basedir}/../../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
</properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- </dependencies>
+ <!-- Do not add dependencies here, add them to the POM of the leaf module -->
<modules>
<module>hadoop-yarn-server-common</module>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml Tue Jan 14 00:01:35 2014
@@ -26,10 +26,13 @@
<artifactId>hadoop-yarn-site</artifactId>
<version>3.0.0-SNAPSHOT</version>
<name>hadoop-yarn-site</name>
+ <packaging>pom</packaging>
<properties>
<!-- Needed for generating FindBugs warnings using parent pom -->
<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
</properties>
+ <!-- Do not add dependencies here, this is a documentation only module -->
+
</project>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Tue Jan 14 00:01:35 2014
@@ -62,6 +62,8 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+
| state | string | The ResourceManager state - valid values are: NOTINITED, INITED, STARTED, STOPPED|
*---------------+--------------+-------------------------------+
+| haState | string | The ResourceManager HA state - valid values are: INITIALIZING, ACTIVE, STANDBY, STOPPED|
+*---------------+--------------+-------------------------------+
| resourceManagerVersion | string | Version of the ResourceManager |
*---------------+--------------+-------------------------------+
| resourceManagerBuildVersion | string | ResourceManager build string with build version, user, and checksum |
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml Tue Jan 14 00:01:35 2014
@@ -33,112 +33,7 @@
<hadoop.common.build.dir>${basedir}/../../../hadoop-common-project/hadoop-common/target</hadoop.common.build.dir>
</properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
- <dependency>
- <groupId>cglib</groupId>
- <artifactId>cglib</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-core</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey.jersey-test-framework</groupId>
- <artifactId>jersey-test-framework-grizzly2</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-json</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </dependency>
- </dependencies>
+ <!-- Do not add dependencies here, add them to the POM of the leaf module -->
<build>
<plugins>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml?rev=1557890&r1=1557889&r2=1557890&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml Tue Jan 14 00:01:35 2014
@@ -41,132 +41,7 @@
<module>hadoop-yarn</module>
</modules>
- <dependencies>
- <dependency>
- <groupId>com.google.protobuf</groupId>
- <artifactId>protobuf-java</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ant</groupId>
- <artifactId>ant</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.velocity</groupId>
- <artifactId>velocity</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <artifactId>paranamer-ant</artifactId>
- <groupId>com.thoughtworks.paranamer</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </dependency>
- <dependency>
- <groupId>cglib</groupId>
- <artifactId>cglib</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey.contribs</groupId>
- <artifactId>jersey-guice</artifactId>
- </dependency>
- <dependency>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </dependency>
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </dependency>
- <dependency>
- <groupId>org.hsqldb</groupId>
- <artifactId>hsqldb</artifactId>
- <scope>compile</scope>
- </dependency>
-
- </dependencies>
+ <!-- Do not add dependencies here, add them to the POM of the leaf module -->
<build>
<plugins>