You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2014/07/07 22:44:16 UTC
svn commit: r1608603 [5/6] - in
/hadoop/common/branches/fs-encryption/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/
had...
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java Mon Jul 7 20:43:56 2014
@@ -80,6 +80,8 @@ public class TestProportionalCapacityPre
static final long TS = 3141592653L;
int appAlloc = 0;
+ boolean setAMContainer = false;
+ float setAMResourcePercent = 0.0f;
Random rand = null;
Clock mClock = null;
Configuration conf = null;
@@ -466,7 +468,108 @@ public class TestProportionalCapacityPre
fail("Failed to find SchedulingMonitor service, please check what happened");
}
+
+ @Test
+ public void testSkipAMContainer() {
+ int[][] qData = new int[][] {
+ // / A B
+ { 100, 50, 50 }, // abs
+ { 100, 100, 100 }, // maxcap
+ { 100, 100, 0 }, // used
+ { 70, 20, 50 }, // pending
+ { 0, 0, 0 }, // reserved
+ { 5, 4, 1 }, // apps
+ { -1, 1, 1 }, // req granularity
+ { 2, 0, 0 }, // subqueues
+ };
+ setAMContainer = true;
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+
+ // By skipping AM Container, all other 24 containers of appD will be
+ // preempted
+ verify(mDisp, times(24)).handle(argThat(new IsPreemptionRequestFor(appD)));
+
+ // By skipping AM Container, all other 24 containers of appC will be
+ // preempted
+ verify(mDisp, times(24)).handle(argThat(new IsPreemptionRequestFor(appC)));
+
+ // Since AM containers of appC and appD are saved, 2 containers from appB
+ // has to be preempted.
+ verify(mDisp, times(2)).handle(argThat(new IsPreemptionRequestFor(appB)));
+ setAMContainer = false;
+ }
+
+ @Test
+ public void testPreemptSkippedAMContainers() {
+ int[][] qData = new int[][] {
+ // / A B
+ { 100, 10, 90 }, // abs
+ { 100, 100, 100 }, // maxcap
+ { 100, 100, 0 }, // used
+ { 70, 20, 90 }, // pending
+ { 0, 0, 0 }, // reserved
+ { 5, 4, 1 }, // apps
+ { -1, 5, 5 }, // req granularity
+ { 2, 0, 0 }, // subqueues
+ };
+ setAMContainer = true;
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+
+ // All 5 containers of appD will be preempted including AM container.
+ verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appD)));
+ // All 5 containers of appC will be preempted including AM container.
+ verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appC)));
+
+ // By skipping AM Container, all other 4 containers of appB will be
+ // preempted
+ verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appB)));
+
+ // By skipping AM Container, all other 4 containers of appA will be
+ // preempted
+ verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appA)));
+ setAMContainer = false;
+ }
+
+ @Test
+ public void testAMResourcePercentForSkippedAMContainers() {
+ int[][] qData = new int[][] {
+ // / A B
+ { 100, 10, 90 }, // abs
+ { 100, 100, 100 }, // maxcap
+ { 100, 100, 0 }, // used
+ { 70, 20, 90 }, // pending
+ { 0, 0, 0 }, // reserved
+ { 5, 4, 1 }, // apps
+ { -1, 5, 5 }, // req granularity
+ { 2, 0, 0 }, // subqueues
+ };
+ setAMContainer = true;
+ setAMResourcePercent = 0.5f;
+ ProportionalCapacityPreemptionPolicy policy = buildPolicy(qData);
+ policy.editSchedule();
+
+ // AMResoucePercent is 50% of cluster and maxAMCapacity will be 5Gb.
+ // Total used AM container size is 20GB, hence 2 AM container has
+ // to be preempted as Queue Capacity is 10Gb.
+ verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appD)));
+
+ // Including AM Container, all other 4 containers of appC will be
+ // preempted
+ verify(mDisp, times(5)).handle(argThat(new IsPreemptionRequestFor(appC)));
+
+ // By skipping AM Container, all other 4 containers of appB will be
+ // preempted
+ verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appB)));
+
+ // By skipping AM Container, all other 4 containers of appA will be
+ // preempted
+ verify(mDisp, times(4)).handle(argThat(new IsPreemptionRequestFor(appA)));
+ setAMContainer = false;
+ }
+
static class IsPreemptionRequestFor
extends ArgumentMatcher<ContainerPreemptEvent> {
private final ApplicationAttemptId appAttId;
@@ -583,6 +686,9 @@ public class TestProportionalCapacityPre
}
}
when(lq.getApplications()).thenReturn(qApps);
+ if(setAMResourcePercent != 0.0f){
+ when(lq.getMaxAMResourcePerQueuePercent()).thenReturn(setAMResourcePercent);
+ }
p.getChildQueues().add(lq);
return lq;
}
@@ -607,7 +713,11 @@ public class TestProportionalCapacityPre
List<RMContainer> cLive = new ArrayList<RMContainer>();
for (int i = 0; i < used; i += gran) {
- cLive.add(mockContainer(appAttId, cAlloc, unit, 1));
+ if(setAMContainer && i == 0){
+ cLive.add(mockContainer(appAttId, cAlloc, unit, 0));
+ }else{
+ cLive.add(mockContainer(appAttId, cAlloc, unit, 1));
+ }
++cAlloc;
}
when(app.getLiveContainers()).thenReturn(cLive);
@@ -623,6 +733,10 @@ public class TestProportionalCapacityPre
RMContainer mC = mock(RMContainer.class);
when(mC.getContainerId()).thenReturn(cId);
when(mC.getContainer()).thenReturn(c);
+ when(mC.getApplicationAttemptId()).thenReturn(appAttId);
+ if(0 == priority){
+ when(mC.isAMContainer()).thenReturn(true);
+ }
return mC;
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java Mon Jul 7 20:43:56 2014
@@ -267,6 +267,7 @@ public class RMStateStoreTestBase extend
// attempt1 is loaded correctly
assertNotNull(attemptState);
assertEquals(attemptId1, attemptState.getAttemptId());
+ assertEquals(-1000, attemptState.getAMContainerExitStatus());
// attempt1 container is loaded correctly
assertEquals(containerId1, attemptState.getMasterContainer().getId());
// attempt1 applicationToken is loaded correctly
@@ -308,7 +309,7 @@ public class RMStateStoreTestBase extend
oldAttemptState.getAppAttemptCredentials(),
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
- FinalApplicationStatus.SUCCEEDED);
+ FinalApplicationStatus.SUCCEEDED, 100);
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -331,7 +332,7 @@ public class RMStateStoreTestBase extend
oldAttemptState.getAppAttemptCredentials(),
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
- FinalApplicationStatus.SUCCEEDED);
+ FinalApplicationStatus.SUCCEEDED, 111);
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
@@ -370,6 +371,7 @@ public class RMStateStoreTestBase extend
assertEquals(RMAppAttemptState.FINISHED, updatedAttemptState.getState());
assertEquals("myTrackingUrl", updatedAttemptState.getFinalTrackingUrl());
assertEquals("attemptDiagnostics", updatedAttemptState.getDiagnostics());
+ assertEquals(100, updatedAttemptState.getAMContainerExitStatus());
assertEquals(FinalApplicationStatus.SUCCEEDED,
updatedAttemptState.getFinalApplicationStatus());
@@ -493,6 +495,21 @@ public class RMStateStoreTestBase extend
Assert.assertTrue(t instanceof RMStateVersionIncompatibleException);
}
}
+
+ public void testEpoch(RMStateStoreHelper stateStoreHelper)
+ throws Exception {
+ RMStateStore store = stateStoreHelper.getRMStateStore();
+ store.setRMDispatcher(new TestDispatcher());
+
+ int firstTimeEpoch = store.getAndIncrementEpoch();
+ Assert.assertEquals(0, firstTimeEpoch);
+
+ int secondTimeEpoch = store.getAndIncrementEpoch();
+ Assert.assertEquals(1, secondTimeEpoch);
+
+ int thirdTimeEpoch = store.getAndIncrementEpoch();
+ Assert.assertEquals(2, thirdTimeEpoch);
+ }
public void testAppDeletion(RMStateStoreHelper stateStoreHelper)
throws Exception {
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java Mon Jul 7 20:43:56 2014
@@ -158,6 +158,7 @@ public class TestFSRMStateStore extends
.getFileSystem(conf).exists(tempAppAttemptFile));
testRMDTSecretManagerStateStore(fsTester);
testCheckVersion(fsTester);
+ testEpoch(fsTester);
testAppDeletion(fsTester);
} finally {
cluster.shutdown();
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Mon Jul 7 20:43:56 2014
@@ -120,6 +120,7 @@ public class TestZKRMStateStore extends
testRMAppStateStore(zkTester);
testRMDTSecretManagerStateStore(zkTester);
testCheckVersion(zkTester);
+ testEpoch(zkTester);
testAppDeletion(zkTester);
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Mon Jul 7 20:43:56 2014
@@ -86,6 +86,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -600,6 +602,9 @@ public class TestRMAppAttemptTransitions
any(List.class),
any(List.class))).
thenReturn(allocation);
+ RMContainer rmContainer = mock(RMContainerImpl.class);
+ when(scheduler.getRMContainer(container.getId())).
+ thenReturn(rmContainer);
applicationAttempt.handle(
new RMAppAttemptContainerAllocatedEvent(
@@ -815,6 +820,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptState());
verifyTokenCount(applicationAttempt.getAppAttemptId(), 1);
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
+ verifyAMCrashAtAllocatedDiagnosticInfo(applicationAttempt.getDiagnostics());
}
@Test
@@ -1232,6 +1238,13 @@ public class TestRMAppAttemptTransitions
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
+ private void verifyAMCrashAtAllocatedDiagnosticInfo(String diagnostics) {
+ assertTrue("Diagnostic information does not contain application proxy URL",
+ diagnostics.contains(applicationAttempt.getWebProxyBase()));
+ assertTrue("Diagnostic information does not point the logs to the users",
+ diagnostics.contains("logs"));
+ }
+
private void verifyTokenCount(ApplicationAttemptId appAttemptId, int count) {
verify(amRMTokenManager, times(count)).applicationMasterFinished(appAttemptId);
if (UserGroupInformation.isSecurityEnabled()) {
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java Mon Jul 7 20:43:56 2014
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.*;
@@ -61,10 +62,15 @@ public class TestSchedulerApplicationAtt
QueueMetrics newMetrics = newQueue.getMetrics();
ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3);
SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId,
- user, oldQueue, oldQueue.getActiveUsersManager(), null);
+ user, oldQueue, oldQueue.getActiveUsersManager(), rmContext);
oldMetrics.submitApp(user);
+ // confirm that containerId is calculated based on epoch.
+ assertEquals(app.getNewContainerId(), 0x00c00001);
+
// Resource request
Resource requestedResource = Resource.newInstance(1536, 2);
Priority requestedPriority = Priority.newInstance(2);
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Mon Jul 7 20:43:56 2014
@@ -25,15 +25,29 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.CyclicBarrier;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.LocalConfigurationProvider;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -46,13 +60,20 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.Task;
+import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
+import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -686,4 +707,125 @@ public class TestCapacityScheduler {
}
}
+ @Test(timeout = 30000)
+ public void testAllocateDoesNotBlockOnSchedulerLock() throws Exception {
+ final YarnConfiguration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ MyContainerManager containerManager = new MyContainerManager();
+ final MockRMWithAMS rm =
+ new MockRMWithAMS(conf, containerManager);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("localhost:1234", 5120);
+
+ Map<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>(2);
+ acls.put(ApplicationAccessType.VIEW_APP, "*");
+ RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
+
+ nm1.nodeHeartbeat(true);
+
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
+ int msecToWait = 10000;
+ int msecToSleep = 100;
+ while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED
+ && msecToWait > 0) {
+ LOG.info("Waiting for AppAttempt to reach LAUNCHED state. "
+ + "Current state is " + attempt.getAppAttemptState());
+ Thread.sleep(msecToSleep);
+ msecToWait -= msecToSleep;
+ }
+ Assert.assertEquals(attempt.getAppAttemptState(),
+ RMAppAttemptState.LAUNCHED);
+
+ // Create a client to the RM.
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ UserGroupInformation currentUser =
+ UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
+ Credentials credentials = containerManager.getContainerCredentials();
+ final InetSocketAddress rmBindAddress =
+ rm.getApplicationMasterService().getBindAddress();
+ Token<? extends TokenIdentifier> amRMToken =
+ MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
+ credentials.getAllTokens());
+ currentUser.addToken(amRMToken);
+ ApplicationMasterProtocol client =
+ currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
+ @Override
+ public ApplicationMasterProtocol run() {
+ return (ApplicationMasterProtocol) rpc.getProxy(
+ ApplicationMasterProtocol.class, rmBindAddress, conf);
+ }
+ });
+
+ RegisterApplicationMasterRequest request =
+ RegisterApplicationMasterRequest.newInstance("localhost", 12345, "");
+ client.registerApplicationMaster(request);
+
+ // grab the scheduler lock from another thread
+ // and verify an allocate call in this thread doesn't block on it
+ final CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ final CyclicBarrier barrier = new CyclicBarrier(2);
+ Thread otherThread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ synchronized(cs) {
+ try {
+ barrier.await();
+ barrier.await();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ } catch (BrokenBarrierException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ });
+ otherThread.start();
+ barrier.await();
+ AllocateRequest allocateRequest =
+ AllocateRequest.newInstance(0, 0.0f, null, null, null);
+ client.allocate(allocateRequest);
+ barrier.await();
+ otherThread.join();
+
+ rm.stop();
+ }
+
+ @Test
+ public void testNumClusterNodes() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacityScheduler cs = new CapacityScheduler();
+ cs.setConf(conf);
+ RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
+ null, new RMContainerTokenSecretManager(conf),
+ new NMTokenSecretManagerInRM(conf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ cs.setRMContext(rmContext);
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(csConf);
+ cs.init(csConf);
+ cs.start();
+ assertEquals(0, cs.getNumClusterNodes());
+
+ RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
+ RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
+ cs.handle(new NodeAddedSchedulerEvent(n1));
+ cs.handle(new NodeAddedSchedulerEvent(n2));
+ assertEquals(2, cs.getNumClusterNodes());
+
+ cs.handle(new NodeRemovedSchedulerEvent(n1));
+ assertEquals(1, cs.getNumClusterNodes());
+ cs.handle(new NodeAddedSchedulerEvent(n1));
+ assertEquals(2, cs.getNumClusterNodes());
+ cs.handle(new NodeRemovedSchedulerEvent(n2));
+ cs.handle(new NodeRemovedSchedulerEvent(n1));
+ assertEquals(0, cs.getNumClusterNodes());
+
+ cs.stop();
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java Mon Jul 7 20:43:56 2014
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.List;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -31,6 +29,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.Token;
@@ -48,6 +47,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -209,10 +209,11 @@ public class TestContainerAllocation {
@Override
public Token createContainerToken(ContainerId containerId,
- NodeId nodeId, String appSubmitter, Resource capability) {
+ NodeId nodeId, String appSubmitter, Resource capability,
+ Priority priority, long createTime) {
numRetries++;
return super.createContainerToken(containerId, nodeId, appSubmitter,
- capability);
+ capability, priority, createTime);
}
};
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java Mon Jul 7 20:43:56 2014
@@ -146,7 +146,7 @@ public class FairSchedulerTestBase {
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
- scheduler.addApplicationAttempt(id, false);
+ scheduler.addApplicationAttempt(id, false, true);
}
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ResourceRequest request = createResourceRequest(memory, vcores, ResourceRequest.ANY,
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSSchedulerApp.java Mon Jul 7 20:43:56 2014
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertEqu
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.util.Clock;
import org.junit.Test;
@@ -59,8 +60,11 @@ public class TestFSSchedulerApp {
double rackLocalityThreshold = .6;
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getEpoch()).thenReturn(0);
FSSchedulerApp schedulerApp =
- new FSSchedulerApp(applicationAttemptId, "user1", queue , null, null);
+ new FSSchedulerApp(applicationAttemptId, "user1", queue , null,
+ rmContext);
// Default level should be node-local
assertEquals(NodeType.NODE_LOCAL, schedulerApp.getAllowedLocalityLevel(
@@ -118,10 +122,12 @@ public class TestFSSchedulerApp {
long nodeLocalityDelayMs = 5 * 1000L; // 5 seconds
long rackLocalityDelayMs = 6 * 1000L; // 6 seconds
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getEpoch()).thenReturn(0);
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
FSSchedulerApp schedulerApp =
new FSSchedulerApp(applicationAttemptId, "user1", queue,
- null, null);
+ null, rmContext);
AppSchedulable appSchedulable = Mockito.mock(AppSchedulable.class);
long startTime = clock.getTime();
Mockito.when(appSchedulable.getStartTime()).thenReturn(startTime);
@@ -173,9 +179,12 @@ public class TestFSSchedulerApp {
Priority prio = Mockito.mock(Priority.class);
Mockito.when(prio.getPriority()).thenReturn(1);
+ RMContext rmContext = Mockito.mock(RMContext.class);
+ Mockito.when(rmContext.getEpoch()).thenReturn(0);
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
FSSchedulerApp schedulerApp =
- new FSSchedulerApp(applicationAttemptId, "user1", queue , null, null);
+ new FSSchedulerApp(applicationAttemptId, "user1", queue , null,
+ rmContext);
assertEquals(NodeType.OFF_SWITCH, schedulerApp.getAllowedLocalityLevel(
prio, 10, -1.0, -1.0));
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon Jul 7 20:43:56 2014
@@ -25,6 +25,8 @@ import static org.junit.Assert.assertNul
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileWriter;
@@ -787,13 +789,13 @@ public class TestFairScheduler extends F
ApplicationAttemptId id11 = createAppAttemptId(1, 1);
scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1");
- scheduler.addApplicationAttempt(id11, false);
+ scheduler.addApplicationAttempt(id11, false, true);
ApplicationAttemptId id21 = createAppAttemptId(2, 1);
scheduler.addApplication(id21.getApplicationId(), "root.queue2", "user1");
- scheduler.addApplicationAttempt(id21, false);
+ scheduler.addApplicationAttempt(id21, false, true);
ApplicationAttemptId id22 = createAppAttemptId(2, 2);
scheduler.addApplication(id22.getApplicationId(), "root.queue2", "user1");
- scheduler.addApplicationAttempt(id22, false);
+ scheduler.addApplicationAttempt(id22, false, true);
int minReqSize =
FairSchedulerConfiguration.DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB;
@@ -1555,7 +1557,7 @@ public class TestFairScheduler extends F
ApplicationAttemptId appId = createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
scheduler.addApplication(appId.getApplicationId(), "queue1", "user1");
- scheduler.addApplicationAttempt(appId, false);
+ scheduler.addApplicationAttempt(appId, false, true);
// 1 request with 2 nodes on the same rack. another request with 1 node on
// a different rack
@@ -1661,6 +1663,41 @@ public class TestFairScheduler extends F
assertEquals("Incorrect number of containers allocated", 8, app
.getLiveContainers().size());
}
+
+ @Test(timeout = 3000)
+ public void testMaxAssignWithZeroMemoryContainers() throws Exception {
+ conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0,
+ "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+
+ ApplicationAttemptId attId =
+ createSchedulingRequest(0, 1, "root.default", "user", 8);
+ FSSchedulerApp app = scheduler.getSchedulerApp(attId);
+
+ // set maxAssign to 2: only 2 containers should be allocated
+ scheduler.maxAssign = 2;
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Incorrect number of containers allocated", 2, app
+ .getLiveContainers().size());
+
+ // set maxAssign to -1: all remaining containers should be allocated
+ scheduler.maxAssign = -1;
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Incorrect number of containers allocated", 8, app
+ .getLiveContainers().size());
+ }
/**
* Test to verify the behavior of
@@ -2484,6 +2521,92 @@ public class TestFairScheduler extends F
}
@Test
+ public void testQueueMaxAMShareDefault() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"queue1\">");
+ out.println("</queue>");
+ out.println("<queue name=\"queue2\">");
+ out.println("<maxAMShare>1.0</maxAMShare>");
+ out.println("</queue>");
+ out.println("<queue name=\"queue3\">");
+ out.println("</queue>");
+ out.println("<queue name=\"queue4\">");
+ out.println("</queue>");
+ out.println("<queue name=\"queue5\">");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node =
+ MockNodes.newNodeInfo(1, Resources.createResource(8192, 20),
+ 0, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
+ scheduler.handle(nodeEvent);
+ scheduler.update();
+
+ FSLeafQueue queue1 =
+ scheduler.getQueueManager().getLeafQueue("queue1", true);
+ assertEquals("Queue queue1's fair share should be 1366",
+ 1366, queue1.getFairShare().getMemory());
+ FSLeafQueue queue2 =
+ scheduler.getQueueManager().getLeafQueue("queue2", true);
+ assertEquals("Queue queue2's fair share should be 1366",
+ 1366, queue2.getFairShare().getMemory());
+ FSLeafQueue queue3 =
+ scheduler.getQueueManager().getLeafQueue("queue3", true);
+ assertEquals("Queue queue3's fair share should be 1366",
+ 1366, queue3.getFairShare().getMemory());
+ FSLeafQueue queue4 =
+ scheduler.getQueueManager().getLeafQueue("queue4", true);
+ assertEquals("Queue queue4's fair share should be 1366",
+ 1366, queue4.getFairShare().getMemory());
+ FSLeafQueue queue5 =
+ scheduler.getQueueManager().getLeafQueue("queue5", true);
+ assertEquals("Queue queue5's fair share should be 1366",
+ 1366, queue5.getFairShare().getMemory());
+
+ Resource amResource1 = Resource.newInstance(2048, 1);
+ int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
+
+ // Exceeds queue limit, but default maxAMShare is -1.0 so it doesn't matter
+ ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
+ createApplicationWithAMResource(attId1, "queue1", "test1", amResource1);
+ createSchedulingRequestExistingApplication(2048, 1, amPriority, attId1);
+ FSSchedulerApp app1 = scheduler.getSchedulerApp(attId1);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application1's AM requests 2048 MB memory",
+ 2048, app1.getAMResource().getMemory());
+ assertEquals("Application1's AM should be running",
+ 1, app1.getLiveContainers().size());
+ assertEquals("Queue1's AM resource usage should be 2048 MB memory",
+ 2048, queue1.getAmResourceUsage().getMemory());
+
+ // Exceeds queue limit, and maxAMShare is 1.0
+ ApplicationAttemptId attId2 = createAppAttemptId(2, 1);
+ createApplicationWithAMResource(attId2, "queue2", "test1", amResource1);
+ createSchedulingRequestExistingApplication(2048, 1, amPriority, attId2);
+ FSSchedulerApp app2 = scheduler.getSchedulerApp(attId2);
+ scheduler.update();
+ scheduler.handle(updateEvent);
+ assertEquals("Application2's AM requests 2048 MB memory",
+ 2048, app2.getAMResource().getMemory());
+ assertEquals("Application2's AM should not be running",
+ 0, app2.getLiveContainers().size());
+ assertEquals("Queue2's AM resource usage should be 0 MB memory",
+ 0, queue2.getAmResourceUsage().getMemory());
+ }
+
+ @Test
public void testMaxRunningAppsHierarchicalQueues() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
MockClock clock = new MockClock();
@@ -2593,7 +2716,7 @@ public class TestFairScheduler extends F
ApplicationAttemptId appAttemptId =
createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
fs.addApplication(appAttemptId.getApplicationId(), "queue11", "user11");
- fs.addApplicationAttempt(appAttemptId, false);
+ fs.addApplicationAttempt(appAttemptId, false, true);
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ResourceRequest request =
createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true);
@@ -2956,4 +3079,71 @@ public class TestFairScheduler extends F
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
+
+ @Test
+ public void testLowestCommonAncestorForNonRootParent() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue aQueue = mock(FSLeafQueue.class);
+ FSLeafQueue bQueue = mock(FSLeafQueue.class);
+ when(aQueue.getName()).thenReturn("root.queue1.a");
+ when(bQueue.getName()).thenReturn("root.queue1.b");
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ FSQueue ancestorQueue =
+ scheduler.findLowestCommonAncestorQueue(aQueue, bQueue);
+ assertEquals(ancestorQueue, queue1);
+ }
+
+ @Test
+ public void testLowestCommonAncestorRootParent() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue aQueue = mock(FSLeafQueue.class);
+ FSLeafQueue bQueue = mock(FSLeafQueue.class);
+ when(aQueue.getName()).thenReturn("root.a");
+ when(bQueue.getName()).thenReturn("root.b");
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("root", false);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ FSQueue ancestorQueue =
+ scheduler.findLowestCommonAncestorQueue(aQueue, bQueue);
+ assertEquals(ancestorQueue, queue1);
+ }
+
+ @Test
+ public void testLowestCommonAncestorDeeperHierarchy() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSQueue aQueue = mock(FSLeafQueue.class);
+ FSQueue bQueue = mock(FSLeafQueue.class);
+ FSQueue a1Queue = mock(FSLeafQueue.class);
+ FSQueue b1Queue = mock(FSLeafQueue.class);
+ when(a1Queue.getName()).thenReturn("root.queue1.a.a1");
+ when(b1Queue.getName()).thenReturn("root.queue1.b.b1");
+ when(aQueue.getChildQueues()).thenReturn(Arrays.asList(a1Queue));
+ when(bQueue.getChildQueues()).thenReturn(Arrays.asList(b1Queue));
+
+ QueueManager queueManager = scheduler.getQueueManager();
+ FSParentQueue queue1 = queueManager.getParentQueue("queue1", true);
+ queue1.addChildQueue(aQueue);
+ queue1.addChildQueue(bQueue);
+
+ FSQueue ancestorQueue =
+ scheduler.findLowestCommonAncestorQueue(a1Queue, b1Queue);
+ assertEquals(ancestorQueue, queue1);
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Mon Jul 7 20:43:56 2014
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.junit.Before;
@@ -40,6 +41,7 @@ public class TestMaxRunningAppsEnforcer
private MaxRunningAppsEnforcer maxAppsEnforcer;
private int appNum;
private TestFairScheduler.MockClock clock;
+ private RMContext rmContext;
@Before
public void setup() throws Exception {
@@ -59,13 +61,16 @@ public class TestMaxRunningAppsEnforcer
userMaxApps = allocConf.userMaxApps;
maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
appNum = 0;
+ rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(0);
}
private FSSchedulerApp addApp(FSLeafQueue queue, String user) {
ApplicationId appId = ApplicationId.newInstance(0l, appNum++);
ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0);
boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user);
- FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null, null);
+ FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null,
+ rmContext);
queue.addApp(app, runnable);
if (runnable) {
maxAppsEnforcer.trackRunnableApp(app);
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java Mon Jul 7 20:43:56 2014
@@ -48,8 +48,8 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
- final int numberOfThInMetricsTable = 13;
- final int numberOfActualTableHeaders = 10;
+ final int numberOfThInMetricsTable = 16;
+ final int numberOfActualTableHeaders = 12;
private Injector injector;
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Mon Jul 7 20:43:56 2014
@@ -389,6 +389,10 @@ public class TestRMWebServices extends J
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
+ WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
WebServicesTestUtils.getXmlInt(element, "totalMB"),
WebServicesTestUtils.getXmlInt(element, "totalNodes"),
@@ -404,11 +408,13 @@ public class TestRMWebServices extends J
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
- assertEquals("incorrect number of elements", 19, clusterinfo.length());
+ assertEquals("incorrect number of elements", 23, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
+ clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
+ clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
clusterinfo.getInt("containersAllocated"),
clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
@@ -418,7 +424,9 @@ public class TestRMWebServices extends J
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
- int allocMB, int containersAlloc, int totalMB, int totalNodes,
+ int allocMB, int reservedVirtualCores, int availableVirtualCores,
+ int allocVirtualCores, int totalVirtualCores,
+ int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@@ -428,7 +436,8 @@ public class TestRMWebServices extends J
long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
-
+ long totalVirtualCoresExpect =
+ metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
@@ -439,6 +448,12 @@ public class TestRMWebServices extends J
metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
+ assertEquals("reservedVirtualCores doesn't match",
+ metrics.getReservedVirtualCores(), reservedVirtualCores);
+ assertEquals("availableVirtualCores doesn't match",
+ metrics.getAvailableVirtualCores(), availableVirtualCores);
+ assertEquals("allocatedVirtualCores doesn't match",
+ totalVirtualCoresExpect, allocVirtualCores);
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals(
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java Mon Jul 7 20:43:56 2014
@@ -20,14 +20,22 @@ package org.apache.hadoop.yarn.server.re
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
+import java.net.URI;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import java.util.Set;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
@@ -37,9 +45,15 @@ import javax.xml.parsers.DocumentBuilder
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -49,10 +63,15 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
@@ -78,6 +97,7 @@ import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.api.client.filter.LoggingFilter;
import com.sun.jersey.api.json.JSONJAXBContext;
import com.sun.jersey.api.json.JSONMarshaller;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
@@ -93,6 +113,8 @@ public class TestRMWebServicesAppsModifi
private Injector injector;
private String webserviceUserName = "testuser";
+ private boolean setAuthFilter = false;
+
public class GuiceServletConfig extends GuiceServletContextListener {
@Override
@@ -131,7 +153,6 @@ public class TestRMWebServicesAppsModifi
private class TestServletModule extends ServletModule {
public Configuration conf = new Configuration();
- boolean setAuthFilter = false;
@Override
protected void configureServlets() {
@@ -157,6 +178,7 @@ public class TestRMWebServicesAppsModifi
return Guice.createInjector(new TestServletModule() {
@Override
protected void configureServlets() {
+ setAuthFilter = false;
super.configureServlets();
}
});
@@ -204,8 +226,8 @@ public class TestRMWebServicesAppsModifi
}
}
- private boolean isAuthorizationEnabled() {
- return rm.getConfig().getBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
+ private boolean isAuthenticationEnabled() {
+ return setAuthFilter;
}
private WebResource constructWebResource(WebResource r, String... paths) {
@@ -213,7 +235,7 @@ public class TestRMWebServicesAppsModifi
for (String path : paths) {
rt = rt.path(path);
}
- if (isAuthorizationEnabled()) {
+ if (isAuthenticationEnabled()) {
rt = rt.queryParam("user.name", webserviceUserName);
}
return rt;
@@ -280,7 +302,7 @@ public class TestRMWebServicesAppsModifi
"state").entity(entity, contentType).accept(mediaType)
.put(ClientResponse.class);
- if (!isAuthorizationEnabled()) {
+ if (!isAuthenticationEnabled()) {
assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
continue;
}
@@ -295,7 +317,7 @@ public class TestRMWebServicesAppsModifi
response.getHeaders().getFirst(HttpHeaders.LOCATION);
Client c = Client.create();
WebResource tmp = c.resource(locationHeaderValue);
- if (isAuthorizationEnabled()) {
+ if (isAuthenticationEnabled()) {
tmp = tmp.queryParam("user.name", webserviceUserName);
}
response = tmp.get(ClientResponse.class);
@@ -361,7 +383,7 @@ public class TestRMWebServicesAppsModifi
.entity(entity, contentType).accept(mediaType)
.put(ClientResponse.class);
- if (!isAuthorizationEnabled()) {
+ if (!isAuthenticationEnabled()) {
assertEquals(Status.UNAUTHORIZED,
response.getClientResponseStatus());
continue;
@@ -426,6 +448,11 @@ public class TestRMWebServicesAppsModifi
@Test(timeout = 30000)
public void testSingleAppKillUnauthorized() throws Exception {
+ boolean isCapacityScheduler =
+ rm.getResourceScheduler() instanceof CapacityScheduler;
+ assumeTrue("Currently this test is only supported on CapacityScheduler",
+ isCapacityScheduler);
+
// default root queue allows anyone to have admin acl
CapacitySchedulerConfiguration csconf =
new CapacitySchedulerConfiguration();
@@ -452,11 +479,7 @@ public class TestRMWebServicesAppsModifi
.constructWebResource("apps", app.getApplicationId().toString(),
"state").accept(mediaType)
.entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class);
- if (!isAuthorizationEnabled()) {
- assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
- } else {
- assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
- }
+ validateResponseStatus(response, Status.FORBIDDEN);
}
rm.stop();
return;
@@ -475,7 +498,7 @@ public class TestRMWebServicesAppsModifi
this.constructWebResource("apps", testAppId, "state")
.accept(MediaType.APPLICATION_XML)
.entity(info, MediaType.APPLICATION_XML).put(ClientResponse.class);
- if (!isAuthorizationEnabled()) {
+ if (!isAuthenticationEnabled()) {
assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
continue;
}
@@ -493,4 +516,348 @@ public class TestRMWebServicesAppsModifi
}
super.tearDown();
}
+
+ /**
+ * Helper function to wrap frequently used code. It checks the response status
+ * and checks if it UNAUTHORIZED if we are running with authorization turned
+ * off or the param passed if we are running with authorization turned on.
+ *
+ * @param response
+ * the ClientResponse object to be checked
+ * @param expectedAuthorizedMode
+ * the expected Status in authorized mode.
+ */
+ public void validateResponseStatus(ClientResponse response,
+ Status expectedAuthorizedMode) {
+ validateResponseStatus(response, Status.UNAUTHORIZED,
+ expectedAuthorizedMode);
+ }
+
+ /**
+ * Helper function to wrap frequently used code. It checks the response status
+ * and checks if it is the param expectedUnauthorizedMode if we are running
+ * with authorization turned off or the param expectedAuthorizedMode passed if
+ * we are running with authorization turned on.
+ *
+ * @param response
+ * the ClientResponse object to be checked
+ * @param expectedUnauthorizedMode
+ * the expected Status in unauthorized mode.
+ * @param expectedAuthorizedMode
+ * the expected Status in authorized mode.
+ */
+ public void validateResponseStatus(ClientResponse response,
+ Status expectedUnauthorizedMode, Status expectedAuthorizedMode) {
+ if (!isAuthenticationEnabled()) {
+ assertEquals(expectedUnauthorizedMode, response.getClientResponseStatus());
+ } else {
+ assertEquals(expectedAuthorizedMode, response.getClientResponseStatus());
+ }
+ }
+
+ // Simple test - just post to /apps/id and validate the response
+ @Test
+ public void testGetNewApplication() throws Exception {
+ // client().addFilter(new LoggingFilter(System.out));
+ rm.start();
+ String mediaTypes[] =
+ { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+ for (String acceptMedia : mediaTypes) {
+ testGetNewApplication(acceptMedia);
+ }
+ rm.stop();
+ return;
+ }
+
+ protected String testGetNewApplication(String mediaType) throws JSONException,
+ ParserConfigurationException, IOException, SAXException {
+ ClientResponse response =
+ this.constructWebResource("apps", "new-application").accept(mediaType)
+ .post(ClientResponse.class);
+ validateResponseStatus(response, Status.OK);
+ if (!isAuthenticationEnabled()) {
+ return "";
+ }
+ return validateGetNewApplicationResponse(response);
+ }
+
+ protected String validateGetNewApplicationResponse(ClientResponse resp)
+ throws JSONException, ParserConfigurationException, IOException,
+ SAXException {
+ String ret = "";
+ if (resp.getType().equals(MediaType.APPLICATION_JSON_TYPE)) {
+ JSONObject json = resp.getEntity(JSONObject.class);
+ ret = validateGetNewApplicationJsonResponse(json);
+ } else if (resp.getType().equals(MediaType.APPLICATION_XML_TYPE)) {
+ String xml = resp.getEntity(String.class);
+ ret = validateGetNewApplicationXMLResponse(xml);
+ } else {
+ // we should not be here
+ assertTrue(false);
+ }
+ return ret;
+ }
+
+ protected String validateGetNewApplicationJsonResponse(JSONObject json)
+ throws JSONException {
+ String appId = json.getString("application-id");
+ assertTrue(appId.isEmpty() == false);
+ JSONObject maxResources = json.getJSONObject("maximum-resource-capability");
+ long memory = maxResources.getLong("memory");
+ long vCores = maxResources.getLong("vCores");
+ assertTrue(memory != 0);
+ assertTrue(vCores != 0);
+ return appId;
+ }
+
+ protected String validateGetNewApplicationXMLResponse(String response)
+ throws ParserConfigurationException, IOException, SAXException {
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(response));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("NewApplication");
+ assertEquals("incorrect number of elements", 1, nodes.getLength());
+ Element element = (Element) nodes.item(0);
+ String appId = WebServicesTestUtils.getXmlString(element, "application-id");
+ assertTrue(appId.isEmpty() == false);
+ NodeList maxResourceNodes =
+ element.getElementsByTagName("maximum-resource-capability");
+ assertEquals(1, maxResourceNodes.getLength());
+ Element maxResourceCapability = (Element) maxResourceNodes.item(0);
+ long memory =
+ WebServicesTestUtils.getXmlLong(maxResourceCapability, "memory");
+ long vCores =
+ WebServicesTestUtils.getXmlLong(maxResourceCapability, "vCores");
+ assertTrue(memory != 0);
+ assertTrue(vCores != 0);
+ return appId;
+ }
+
+ // Test to validate the process of submitting apps - test for appropriate
+ // errors as well
+ @Test
+ public void testGetNewApplicationAndSubmit() throws Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+ String mediaTypes[] =
+ { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+ for (String acceptMedia : mediaTypes) {
+ for (String contentMedia : mediaTypes) {
+ testAppSubmit(acceptMedia, contentMedia);
+ testAppSubmitErrors(acceptMedia, contentMedia);
+ }
+ }
+ rm.stop();
+ return;
+ }
+
+ public void testAppSubmit(String acceptMedia, String contentMedia)
+ throws Exception {
+
+ // create a test app and submit it via rest(after getting an app-id) then
+ // get the app details from the rmcontext and check that everything matches
+
+ // client().addFilter(new LoggingFilter(System.out));
+ String lrKey = "example";
+ String queueName = "testqueue";
+ String appName = "test";
+ String appType = "test-type";
+ String urlPath = "apps";
+ String appId = testGetNewApplication(acceptMedia);
+ List<String> commands = new ArrayList<String>();
+ commands.add("/bin/sleep 5");
+ HashMap<String, String> environment = new HashMap<String, String>();
+ environment.put("APP_VAR", "ENV_SETTING");
+ HashMap<ApplicationAccessType, String> acls =
+ new HashMap<ApplicationAccessType, String>();
+ acls.put(ApplicationAccessType.MODIFY_APP, "testuser1, testuser2");
+ acls.put(ApplicationAccessType.VIEW_APP, "testuser3, testuser4");
+ Set<String> tags = new HashSet<String>();
+ tags.add("tag1");
+ tags.add("tag 2");
+ CredentialsInfo credentials = new CredentialsInfo();
+ HashMap<String, String> tokens = new HashMap<String, String>();
+ HashMap<String, String> secrets = new HashMap<String, String>();
+ secrets.put("secret1", Base64.encodeBase64URLSafeString("secret1".getBytes("UTF8")));
+ credentials.setSecrets(secrets);
+ credentials.setTokens(tokens);
+ ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo();
+ appInfo.setApplicationId(appId);
+ appInfo.setApplicationName(appName);
+ appInfo.setPriority(3);
+ appInfo.setMaxAppAttempts(2);
+ appInfo.setQueue(queueName);
+ appInfo.setApplicationType(appType);
+ HashMap<String, LocalResourceInfo> lr =
+ new HashMap<String, LocalResourceInfo>();
+ LocalResourceInfo y = new LocalResourceInfo();
+ y.setUrl(new URI("http://www.test.com/file.txt"));
+ y.setSize(100);
+ y.setTimestamp(System.currentTimeMillis());
+ y.setType(LocalResourceType.FILE);
+ y.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.put(lrKey, y);
+ appInfo.getContainerLaunchContextInfo().setResources(lr);
+ appInfo.getContainerLaunchContextInfo().setCommands(commands);
+ appInfo.getContainerLaunchContextInfo().setEnvironment(environment);
+ appInfo.getContainerLaunchContextInfo().setAcls(acls);
+ appInfo.getContainerLaunchContextInfo().getAuxillaryServiceData()
+ .put("test", Base64.encodeBase64URLSafeString("value12".getBytes("UTF8")));
+ appInfo.getContainerLaunchContextInfo().setCredentials(credentials);
+ appInfo.getResource().setMemory(1024);
+ appInfo.getResource().setvCores(1);
+ appInfo.setApplicationTags(tags);
+
+ ClientResponse response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+
+ if (this.isAuthenticationEnabled() == false) {
+ assertEquals(Status.UNAUTHORIZED, response.getClientResponseStatus());
+ return;
+ }
+ assertEquals(Status.ACCEPTED, response.getClientResponseStatus());
+ assertTrue(response.getHeaders().getFirst(HttpHeaders.LOCATION).isEmpty() == false);
+ String locURL = response.getHeaders().getFirst(HttpHeaders.LOCATION);
+ assertTrue(locURL.indexOf("/apps/application") != -1);
+ appId = locURL.substring(locURL.indexOf("/apps/") + "/apps/".length());
+
+ WebResource res = resource().uri(new URI(locURL));
+ res = res.queryParam("user.name", webserviceUserName);
+ response = res.get(ClientResponse.class);
+ assertEquals(Status.OK, response.getClientResponseStatus());
+
+ RMApp app =
+ rm.getRMContext().getRMApps()
+ .get(ConverterUtils.toApplicationId(appId));
+ assertEquals(appName, app.getName());
+ assertEquals(webserviceUserName, app.getUser());
+ assertEquals(2, app.getMaxAppAttempts());
+ assertEquals(queueName, app.getQueue());
+ assertEquals(appType, app.getApplicationType());
+ assertEquals(tags, app.getApplicationTags());
+ ContainerLaunchContext ctx =
+ app.getApplicationSubmissionContext().getAMContainerSpec();
+ assertEquals(commands, ctx.getCommands());
+ assertEquals(environment, ctx.getEnvironment());
+ assertEquals(acls, ctx.getApplicationACLs());
+ Map<String, LocalResource> appLRs = ctx.getLocalResources();
+ assertTrue(appLRs.containsKey(lrKey));
+ LocalResource exampleLR = appLRs.get(lrKey);
+ assertEquals(ConverterUtils.getYarnUrlFromURI(y.getUrl()),
+ exampleLR.getResource());
+ assertEquals(y.getSize(), exampleLR.getSize());
+ assertEquals(y.getTimestamp(), exampleLR.getTimestamp());
+ assertEquals(y.getType(), exampleLR.getType());
+ assertEquals(y.getPattern(), exampleLR.getPattern());
+ assertEquals(y.getVisibility(), exampleLR.getVisibility());
+
+ response =
+ this.constructWebResource("apps", appId).accept(acceptMedia)
+ .get(ClientResponse.class);
+ assertEquals(Status.OK, response.getClientResponseStatus());
+ return;
+ }
+
+ public void testAppSubmitErrors(String acceptMedia, String contentMedia)
+ throws Exception {
+
+ // submit a bunch of bad requests(correct format but bad values) via the
+ // REST API and make sure we get the right error response codes
+
+ String urlPath = "apps";
+ String appId = "";
+ ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo();
+ ClientResponse response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ appId = "random";
+ appInfo.setApplicationId(appId);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ appId = "random_junk";
+ appInfo.setApplicationId(appId);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ // bad resource info
+ appInfo.getResource().setMemory(
+ rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB) + 1);
+ appInfo.getResource().setvCores(1);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ appInfo.getResource().setvCores(
+ rm.getConfig().getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES) + 1);
+ appInfo.getResource().setMemory(CONTAINER_MB);
+ response =
+ this.constructWebResource(urlPath).accept(acceptMedia)
+ .entity(appInfo, contentMedia).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+
+ return;
+ }
+
+ @Test
+ public void testAppSubmitBadJsonAndXML() throws Exception {
+
+ // submit a bunch of bad XML and JSON via the
+ // REST API and make sure we get error response codes
+
+ String urlPath = "apps";
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+
+ ApplicationSubmissionContextInfo appInfo = new ApplicationSubmissionContextInfo();
+ appInfo.setApplicationName("test");
+ appInfo.setPriority(3);
+ appInfo.setMaxAppAttempts(2);
+ appInfo.setQueue("testqueue");
+ appInfo.setApplicationType("test-type");
+ HashMap<String, LocalResourceInfo> lr =
+ new HashMap<String, LocalResourceInfo>();
+ LocalResourceInfo y = new LocalResourceInfo();
+ y.setUrl(new URI("http://www.test.com/file.txt"));
+ y.setSize(100);
+ y.setTimestamp(System.currentTimeMillis());
+ y.setType(LocalResourceType.FILE);
+ y.setVisibility(LocalResourceVisibility.APPLICATION);
+ lr.put("example", y);
+ appInfo.getContainerLaunchContextInfo().setResources(lr);
+ appInfo.getResource().setMemory(1024);
+ appInfo.getResource().setvCores(1);
+
+ String body =
+ "<?xml version=\"1.0\" encoding=\"UTF-8\" "
+ + "standalone=\"yes\"?><blah/>";
+ ClientResponse response =
+ this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML)
+ .entity(body, MediaType.APPLICATION_XML).post(ClientResponse.class);
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ body = "{\"a\" : \"b\"}";
+ response =
+ this.constructWebResource(urlPath).accept(MediaType.APPLICATION_XML)
+ .entity(body, MediaType.APPLICATION_JSON).post(ClientResponse.class);
+ validateResponseStatus(response, Status.BAD_REQUEST);
+ rm.stop();
+ }
+
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Mon Jul 7 20:43:56 2014
@@ -656,13 +656,15 @@ public class TestRMWebServicesNodes exte
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
+ WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
+ WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 11, nodeInfo.length());
+ assertEquals("incorrect number of elements", 13, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@@ -671,6 +673,7 @@ public class TestRMWebServicesNodes exte
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
+ nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
nodeInfo.getString("version"));
}
@@ -678,7 +681,8 @@ public class TestRMWebServicesNodes exte
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
- int numContainers, long usedMemoryMB, long availMemoryMB, String version)
+ int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
+ long availVirtualCores, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@@ -712,6 +716,10 @@ public class TestRMWebServicesNodes exte
.getUsedResource().getMemory(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
.getAvailableResource().getMemory(), availMemoryMB);
+ assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
+ .getUsedResource().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
+ .getAvailableResource().getVirtualCores(), availVirtualCores);
}
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1608603&r1=1608602&r2=1608603&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Mon Jul 7 20:43:56 2014
@@ -29,8 +29,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -52,6 +50,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
@@ -60,15 +59,18 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -200,8 +202,6 @@ public class TestContainerManagerSecurit
ApplicationId appId = ApplicationId.newInstance(1, 1);
ApplicationAttemptId validAppAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
- ApplicationAttemptId invalidAppAttemptId =
- ApplicationAttemptId.newInstance(appId, 2);
ContainerId validContainerId =
ContainerId.newInstance(validAppAttemptId, 0);
@@ -215,7 +215,11 @@ public class TestContainerManagerSecurit
org.apache.hadoop.yarn.api.records.Token validContainerToken =
containerTokenSecretManager.createContainerToken(validContainerId,
- validNode, user, r);
+ validNode, user, r, Priority.newInstance(10), 1234);
+ ContainerTokenIdentifier identifier =
+ BuilderUtils.newContainerTokenIdentifier(validContainerToken);
+ Assert.assertEquals(Priority.newInstance(10), identifier.getPriority());
+ Assert.assertEquals(1234, identifier.getCreationTime());
StringBuilder sb;
// testInvalidNMToken ... creating NMToken using different secret manager.
@@ -263,27 +267,14 @@ public class TestContainerManagerSecurit
testStartContainer(rpc, validAppAttemptId, validNode,
validContainerToken, invalidNMToken, true)));
- // using appAttempt-2 token for launching container for appAttempt-1.
- invalidNMToken =
- nmTokenSecretManagerRM.createNMToken(invalidAppAttemptId, validNode,
- user);
- sb = new StringBuilder("\nNMToken for application attempt : ");
- sb.append(invalidAppAttemptId.toString())
- .append(" was used for starting container with container token")
- .append(" issued for application attempt : ")
- .append(validAppAttemptId.toString());
- Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
- validContainerToken, invalidNMToken, true).contains(sb.toString()));
-
// using correct tokens. nmtoken for app attempt should get saved.
conf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
4 * 60 * 1000);
validContainerToken =
containerTokenSecretManager.createContainerToken(validContainerId,
- validNode, user, r);
-
- testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken,
- validNMToken, false);
+ validNode, user, r, Priority.newInstance(0), 0);
+ Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, validNMToken, false).isEmpty());
Assert.assertTrue(nmTokenSecretManagerNM
.isAppAttemptNMTokenKeyPresent(validAppAttemptId));
@@ -325,6 +316,18 @@ public class TestContainerManagerSecurit
Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, false).contains(sb.toString()));
+ // using appAttempt-1 NMtoken for launching container for appAttempt-2 should
+ // succeed.
+ ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(appId, 2);
+ Token attempt1NMToken =
+ nmTokenSecretManagerRM
+ .createNMToken(validAppAttemptId, validNode, user);
+ org.apache.hadoop.yarn.api.records.Token newContainerToken =
+ containerTokenSecretManager.createContainerToken(
+ ContainerId.newInstance(attempt2, 1), validNode, user, r,
+ Priority.newInstance(0), 0);
+ Assert.assertTrue(testStartContainer(rpc, attempt2, validNode,
+ newContainerToken, attempt1NMToken, false).isEmpty());
}
private void waitForContainerToFinishOnNM(ContainerId containerId) {
@@ -590,7 +593,7 @@ public class TestContainerManagerSecurit
// Creating modified containerToken
Token containerToken =
tamperedContainerTokenSecretManager.createContainerToken(cId, nodeId,
- user, r);
+ user, r, Priority.newInstance(0), 0);
Token nmToken =
nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.create(conf);