You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/08/12 23:26:19 UTC
svn commit: r1513258 [3/10] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/conf/ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/async/impl/TestAMRMClientAsync.java Mon Aug 12 21:25:49 2013
@@ -21,10 +21,12 @@ package org.apache.hadoop.yarn.client.ap
import static org.mockito.Matchers.anyFloat;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.spy;
import java.io.IOException;
import java.util.ArrayList;
@@ -54,6 +56,7 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -156,14 +159,26 @@ public class TestAMRMClientAsync {
@Test(timeout=10000)
public void testAMRMClientAsyncException() throws Exception {
+ String exStr = "TestException";
+ YarnException mockException = mock(YarnException.class);
+ when(mockException.getMessage()).thenReturn(exStr);
+ runHeartBeatThrowOutException(mockException);
+ }
+
+ @Test(timeout=10000)
+ public void testAMRMClientAsyncRunTimeException() throws Exception {
+ String exStr = "TestRunTimeException";
+ RuntimeException mockRunTimeException = mock(RuntimeException.class);
+ when(mockRunTimeException.getMessage()).thenReturn(exStr);
+ runHeartBeatThrowOutException(mockRunTimeException);
+ }
+
+ private void runHeartBeatThrowOutException(Exception ex) throws Exception{
Configuration conf = new Configuration();
TestCallbackHandler callbackHandler = new TestCallbackHandler();
@SuppressWarnings("unchecked")
AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
- String exStr = "TestException";
- YarnException mockException = mock(YarnException.class);
- when(mockException.getMessage()).thenReturn(exStr);
- when(client.allocate(anyFloat())).thenThrow(mockException);
+ when(client.allocate(anyFloat())).thenThrow(ex);
AMRMClientAsync<ContainerRequest> asyncClient =
AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
@@ -180,14 +195,14 @@ public class TestAMRMClientAsync {
}
}
}
-
- Assert.assertTrue(callbackHandler.savedException.getMessage().contains(exStr));
+ Assert.assertTrue(callbackHandler.savedException.getMessage().contains(
+ ex.getMessage()));
asyncClient.stop();
// stopping should have joined all threads and completed all callbacks
Assert.assertTrue(callbackHandler.callbackCount == 0);
}
-
+
@Test//(timeout=10000)
public void testAMRMClientAsyncReboot() throws Exception {
Configuration conf = new Configuration();
@@ -264,13 +279,13 @@ public class TestAMRMClientAsync {
AMRMClientAsync<ContainerRequest> asyncClient =
AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
- callbackHandler.registerAsyncClient(asyncClient);
+ callbackHandler.asynClient = asyncClient;
asyncClient.init(conf);
asyncClient.start();
synchronized (callbackHandler.notifier) {
asyncClient.registerApplicationMaster("localhost", 1234, null);
- while(callbackHandler.stop == false) {
+ while(callbackHandler.notify == false) {
try {
callbackHandler.notifier.wait();
} catch (InterruptedException e) {
@@ -280,6 +295,65 @@ public class TestAMRMClientAsync {
}
}
+ void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws
+ InterruptedException, YarnException, IOException {
+ Configuration conf = new Configuration();
+ @SuppressWarnings("unchecked")
+ AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
+
+ List<ContainerStatus> completed = Arrays.asList(
+ ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
+ ContainerState.COMPLETE, "", 0));
+ final AllocateResponse response = createAllocateResponse(completed,
+ new ArrayList<Container>(), null);
+
+ when(client.allocate(anyFloat())).thenReturn(response);
+ AMRMClientAsync<ContainerRequest> asyncClient =
+ AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
+ callbackHandler.asynClient = asyncClient;
+ callbackHandler.throwOutException = true;
+ asyncClient.init(conf);
+ asyncClient.start();
+
+ // call register and wait for error callback and stop
+ synchronized (callbackHandler.notifier) {
+ asyncClient.registerApplicationMaster("localhost", 1234, null);
+ while(callbackHandler.notify == false) {
+ try {
+ callbackHandler.notifier.wait();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ // verify error invoked
+ verify(callbackHandler, times(0)).getProgress();
+ verify(callbackHandler, times(1)).onError(any(Exception.class));
+ // sleep to wait for a few heartbeat calls that can trigger callbacks
+ Thread.sleep(50);
+ // verify no more invocations after the first one.
+ // ie. callback thread has stopped
+ verify(callbackHandler, times(0)).getProgress();
+ verify(callbackHandler, times(1)).onError(any(Exception.class));
+ }
+
+ @Test (timeout = 5000)
+ public void testCallBackThrowOutException() throws YarnException,
+ IOException, InterruptedException {
+ // test exception in callback with app calling stop() on app.onError()
+ TestCallbackHandler2 callbackHandler = spy(new TestCallbackHandler2());
+ runCallBackThrowOutException(callbackHandler);
+ }
+
+ @Test (timeout = 5000)
+ public void testCallBackThrowOutExceptionNoStop() throws YarnException,
+ IOException, InterruptedException {
+ // test exception in callback with app not calling stop() on app.onError()
+ TestCallbackHandler2 callbackHandler = spy(new TestCallbackHandler2());
+ callbackHandler.stop = false;
+ runCallBackThrowOutException(callbackHandler);
+ }
+
private AllocateResponse createAllocateResponse(
List<ContainerStatus> completed, List<Container> allocated,
List<NMToken> nmTokens) {
@@ -378,8 +452,8 @@ public class TestAMRMClientAsync {
}
@Override
- public void onError(Exception e) {
- savedException = e;
+ public void onError(Throwable e) {
+ savedException = new Exception(e.getMessage());
synchronized (notifier) {
notifier.notifyAll();
}
@@ -390,10 +464,16 @@ public class TestAMRMClientAsync {
Object notifier = new Object();
@SuppressWarnings("rawtypes")
AMRMClientAsync asynClient;
- boolean stop = false;
+ boolean stop = true;
+ boolean notify = false;
+ boolean throwOutException = false;
@Override
- public void onContainersCompleted(List<ContainerStatus> statuses) {}
+ public void onContainersCompleted(List<ContainerStatus> statuses) {
+ if (throwOutException) {
+ throw new YarnRuntimeException("Exception from callback handler");
+ }
+ }
@Override
public void onContainersAllocated(List<Container> containers) {}
@@ -406,20 +486,24 @@ public class TestAMRMClientAsync {
@Override
public float getProgress() {
- asynClient.stop();
- stop = true;
- synchronized (notifier) {
- notifier.notifyAll();
- }
+ callStopAndNotify();
return 0;
}
@Override
- public void onError(Exception e) {}
+ public void onError(Throwable e) {
+ Assert.assertEquals(e.getMessage(), "Exception from callback handler");
+ callStopAndNotify();
+ }
- public void registerAsyncClient(
- AMRMClientAsync<ContainerRequest> asyncClient) {
- this.asynClient = asyncClient;
+ void callStopAndNotify() {
+ if(stop) {
+ asynClient.stop();
+ }
+ notify = true;
+ synchronized (notifier) {
+ notifier.notifyAll();
+ }
}
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Mon Aug 12 21:25:49 2013
@@ -24,7 +24,10 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Set;
@@ -33,11 +36,13 @@ import java.util.TreeSet;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -48,6 +53,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
@@ -57,13 +63,15 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
-import org.apache.hadoop.yarn.client.api.AMRMClient.StoredContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.AfterClass;
@@ -72,6 +80,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
+import org.mortbay.log.Log;
public class TestAMRMClient {
static Configuration conf = null;
@@ -93,6 +102,7 @@ public class TestAMRMClient {
public static void setup() throws Exception {
// start minicluster
conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
@@ -130,11 +140,14 @@ public class TestAMRMClient {
// Set the queue to which this application is to be submitted in the RM
appContext.setQueue("default");
// Set up the container launch context for the application master
- ContainerLaunchContext amContainer = Records
- .newRecord(ContainerLaunchContext.class);
+ ContainerLaunchContext amContainer =
+ BuilderUtils.newContainerLaunchContext(
+ Collections.<String, LocalResource> emptyMap(),
+ new HashMap<String, String>(), Arrays.asList("sleep", "100"),
+ new HashMap<String, ByteBuffer>(), null,
+ new HashMap<ApplicationAccessType, String>());
appContext.setAMContainerSpec(amContainer);
- // unmanaged AM
- appContext.setUnmanagedAM(true);
+ appContext.setResource(Resource.newInstance(1024, 1));
// Create the request to send to the applications manager
SubmitApplicationRequest appRequest = Records
.newRecord(SubmitApplicationRequest.class);
@@ -143,17 +156,32 @@ public class TestAMRMClient {
yarnClient.submitApplication(appContext);
// wait for app to start
+ RMAppAttempt appAttempt = null;
while (true) {
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
attemptId = appReport.getCurrentApplicationAttemptId();
+ appAttempt =
+ yarnCluster.getResourceManager().getRMContext().getRMApps()
+ .get(attemptId.getApplicationId()).getCurrentAppAttempt();
+ while (true) {
+ if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
+ break;
+ }
+ }
break;
}
}
+ // Just dig into the ResourceManager and get the AMRMToken just for the sake
+ // of testing.
+ UserGroupInformation.setLoginUser(UserGroupInformation
+ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
+ UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
}
@After
- public void cancelApp() {
+ public void cancelApp() throws YarnException, IOException {
+ yarnClient.killApplication(attemptId.getApplicationId());
attemptId = null;
}
@@ -169,10 +197,10 @@ public class TestAMRMClient {
@Test (timeout=60000)
public void testAMRMClientMatchingFit() throws YarnException, IOException {
- AMRMClient<StoredContainerRequest> amClient = null;
+ AMRMClient<ContainerRequest> amClient = null;
try {
// start am rm client
- amClient = AMRMClient.<StoredContainerRequest>createAMRMClient(attemptId);
+ amClient = AMRMClient.<ContainerRequest>createAMRMClient();
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
@@ -185,20 +213,20 @@ public class TestAMRMClient {
Resource capability6 = Resource.newInstance(2000, 1);
Resource capability7 = Resource.newInstance(2000, 1);
- StoredContainerRequest storedContainer1 =
- new StoredContainerRequest(capability1, nodes, racks, priority);
- StoredContainerRequest storedContainer2 =
- new StoredContainerRequest(capability2, nodes, racks, priority);
- StoredContainerRequest storedContainer3 =
- new StoredContainerRequest(capability3, nodes, racks, priority);
- StoredContainerRequest storedContainer4 =
- new StoredContainerRequest(capability4, nodes, racks, priority);
- StoredContainerRequest storedContainer5 =
- new StoredContainerRequest(capability5, nodes, racks, priority);
- StoredContainerRequest storedContainer6 =
- new StoredContainerRequest(capability6, nodes, racks, priority);
- StoredContainerRequest storedContainer7 =
- new StoredContainerRequest(capability7, nodes, racks, priority2, false);
+ ContainerRequest storedContainer1 =
+ new ContainerRequest(capability1, nodes, racks, priority);
+ ContainerRequest storedContainer2 =
+ new ContainerRequest(capability2, nodes, racks, priority);
+ ContainerRequest storedContainer3 =
+ new ContainerRequest(capability3, nodes, racks, priority);
+ ContainerRequest storedContainer4 =
+ new ContainerRequest(capability4, nodes, racks, priority);
+ ContainerRequest storedContainer5 =
+ new ContainerRequest(capability5, nodes, racks, priority);
+ ContainerRequest storedContainer6 =
+ new ContainerRequest(capability6, nodes, racks, priority);
+ ContainerRequest storedContainer7 =
+ new ContainerRequest(capability7, nodes, racks, priority2, false);
amClient.addContainerRequest(storedContainer1);
amClient.addContainerRequest(storedContainer2);
amClient.addContainerRequest(storedContainer3);
@@ -208,8 +236,8 @@ public class TestAMRMClient {
amClient.addContainerRequest(storedContainer7);
// test matching of containers
- List<? extends Collection<StoredContainerRequest>> matches;
- StoredContainerRequest storedRequest;
+ List<? extends Collection<ContainerRequest>> matches;
+ ContainerRequest storedRequest;
// exact match
Resource testCapability1 = Resource.newInstance(1024, 2);
matches = amClient.getMatchingRequests(priority, node, testCapability1);
@@ -224,7 +252,7 @@ public class TestAMRMClient {
verifyMatches(matches, 2);
// must be returned in the order they were made
int i = 0;
- for(StoredContainerRequest storedRequest1 : matches.get(0)) {
+ for(ContainerRequest storedRequest1 : matches.get(0)) {
if(i++ == 0) {
assertTrue(storedContainer4 == storedRequest1);
} else {
@@ -242,9 +270,9 @@ public class TestAMRMClient {
matches = amClient.getMatchingRequests(priority, node, testCapability4);
assert(matches.size() == 2);
// verify non-fitting containers are not returned and fitting ones are
- for(Collection<StoredContainerRequest> testSet : matches) {
+ for(Collection<ContainerRequest> testSet : matches) {
assertTrue(testSet.size() == 1);
- StoredContainerRequest testRequest = testSet.iterator().next();
+ ContainerRequest testRequest = testSet.iterator().next();
assertTrue(testRequest != storedContainer4);
assertTrue(testRequest != storedContainer5);
assert(testRequest == storedContainer2 ||
@@ -275,7 +303,7 @@ public class TestAMRMClient {
}
private void verifyMatches(
- List<? extends Collection<StoredContainerRequest>> matches,
+ List<? extends Collection<ContainerRequest>> matches,
int matchSize) {
assertTrue(matches.size() == 1);
assertTrue(matches.get(0).size() == matchSize);
@@ -283,23 +311,23 @@ public class TestAMRMClient {
@Test (timeout=60000)
public void testAMRMClientMatchingFitInferredRack() throws YarnException, IOException {
- AMRMClientImpl<StoredContainerRequest> amClient = null;
+ AMRMClientImpl<ContainerRequest> amClient = null;
try {
// start am rm client
- amClient = new AMRMClientImpl<StoredContainerRequest>(attemptId);
+ amClient = new AMRMClientImpl<ContainerRequest>();
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
Resource capability = Resource.newInstance(1024, 2);
- StoredContainerRequest storedContainer1 =
- new StoredContainerRequest(capability, nodes, null, priority);
+ ContainerRequest storedContainer1 =
+ new ContainerRequest(capability, nodes, null, priority);
amClient.addContainerRequest(storedContainer1);
// verify matching with original node and inferred rack
- List<? extends Collection<StoredContainerRequest>> matches;
- StoredContainerRequest storedRequest;
+ List<? extends Collection<ContainerRequest>> matches;
+ ContainerRequest storedRequest;
// exact match node
matches = amClient.getMatchingRequests(priority, node, capability);
verifyMatches(matches, 1);
@@ -326,14 +354,14 @@ public class TestAMRMClient {
}
}
- @Test (timeout=60000)
+ @Test //(timeout=60000)
public void testAMRMClientMatchStorage() throws YarnException, IOException {
- AMRMClientImpl<StoredContainerRequest> amClient = null;
+ AMRMClientImpl<ContainerRequest> amClient = null;
try {
// start am rm client
amClient =
- (AMRMClientImpl<StoredContainerRequest>) AMRMClient
- .<StoredContainerRequest> createAMRMClient(attemptId);
+ (AMRMClientImpl<ContainerRequest>) AMRMClient
+ .<ContainerRequest> createAMRMClient();
amClient.init(conf);
amClient.start();
amClient.registerApplicationMaster("Host", 10000, "");
@@ -341,12 +369,12 @@ public class TestAMRMClient {
Priority priority1 = Records.newRecord(Priority.class);
priority1.setPriority(2);
- StoredContainerRequest storedContainer1 =
- new StoredContainerRequest(capability, nodes, racks, priority);
- StoredContainerRequest storedContainer2 =
- new StoredContainerRequest(capability, nodes, racks, priority);
- StoredContainerRequest storedContainer3 =
- new StoredContainerRequest(capability, null, null, priority1);
+ ContainerRequest storedContainer1 =
+ new ContainerRequest(capability, nodes, racks, priority);
+ ContainerRequest storedContainer2 =
+ new ContainerRequest(capability, nodes, racks, priority);
+ ContainerRequest storedContainer3 =
+ new ContainerRequest(capability, null, null, priority1);
amClient.addContainerRequest(storedContainer1);
amClient.addContainerRequest(storedContainer2);
amClient.addContainerRequest(storedContainer3);
@@ -358,7 +386,7 @@ public class TestAMRMClient {
containersRequestedAny = amClient.remoteRequestsTable.get(priority1)
.get(ResourceRequest.ANY).get(capability).remoteRequest.getNumContainers();
assertTrue(containersRequestedAny == 1);
- List<? extends Collection<StoredContainerRequest>> matches =
+ List<? extends Collection<ContainerRequest>> matches =
amClient.getMatchingRequests(priority, node, capability);
verifyMatches(matches, 2);
matches = amClient.getMatchingRequests(priority, rack, capability);
@@ -383,7 +411,7 @@ public class TestAMRMClient {
verifyMatches(matches, 1);
// test matching of containers
- StoredContainerRequest storedRequest = matches.get(0).iterator().next();
+ ContainerRequest storedRequest = matches.get(0).iterator().next();
assertTrue(storedContainer1 == storedRequest);
amClient.removeContainerRequest(storedContainer1);
matches =
@@ -400,9 +428,10 @@ public class TestAMRMClient {
amClient.addContainerRequest(storedContainer3);
// RM should allocate container within 2 calls to allocate()
int allocatedContainerCount = 0;
- int iterationsLeft = 2;
+ int iterationsLeft = 3;
while (allocatedContainerCount < 2
&& iterationsLeft-- > 0) {
+ Log.info(" == alloc " + allocatedContainerCount + " it left " + iterationsLeft);
AllocateResponse allocResponse = amClient.allocate(0.1f);
assertTrue(amClient.ask.size() == 0);
assertTrue(amClient.release.size() == 0);
@@ -420,24 +449,23 @@ public class TestAMRMClient {
verifyMatches(matches, 1);
ContainerRequest matchedRequest = matches.get(0).iterator().next();
assertTrue(matchedRequest == expectedRequest);
-
+ amClient.removeContainerRequest(matchedRequest);
// assign this container, use it and release it
amClient.releaseAssignedContainer(container.getId());
}
if(allocatedContainerCount < containersRequestedAny) {
// sleep to let NM's heartbeat to RM and trigger allocations
- sleep(1000);
+ sleep(100);
}
}
assertTrue(allocatedContainerCount == 2);
- assertTrue(amClient.release.size() == 2);
- assertTrue(amClient.ask.size() == 0);
AllocateResponse allocResponse = amClient.allocate(0.1f);
assertTrue(amClient.release.size() == 0);
assertTrue(amClient.ask.size() == 0);
assertTrue(allocResponse.getAllocatedContainers().size() == 0);
-
+ // 0 requests left. everything got cleaned up
+ assertTrue(amClient.remoteRequestsTable.isEmpty());
amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@ -454,7 +482,7 @@ public class TestAMRMClient {
AMRMClient<ContainerRequest> amClient = null;
try {
// start am rm client
- amClient = AMRMClient.<ContainerRequest>createAMRMClient(attemptId);
+ amClient = AMRMClient.<ContainerRequest>createAMRMClient();
amClient.init(conf);
amClient.start();
@@ -480,11 +508,17 @@ public class TestAMRMClient {
assertTrue(amClient.release.size() == 0);
amClient.addContainerRequest(
- new ContainerRequest(capability, nodes, racks, priority, 1));
+ new ContainerRequest(capability, nodes, racks, priority));
+ amClient.addContainerRequest(
+ new ContainerRequest(capability, nodes, racks, priority));
amClient.addContainerRequest(
- new ContainerRequest(capability, nodes, racks, priority, 3));
+ new ContainerRequest(capability, nodes, racks, priority));
+ amClient.addContainerRequest(
+ new ContainerRequest(capability, nodes, racks, priority));
+ amClient.removeContainerRequest(
+ new ContainerRequest(capability, nodes, racks, priority));
amClient.removeContainerRequest(
- new ContainerRequest(capability, nodes, racks, priority, 2));
+ new ContainerRequest(capability, nodes, racks, priority));
int containersRequestedNode = amClient.remoteRequestsTable.get(priority)
.get(node).get(capability).remoteRequest.getNumContainers();
@@ -501,7 +535,7 @@ public class TestAMRMClient {
// RM should allocate container within 2 calls to allocate()
int allocatedContainerCount = 0;
- int iterationsLeft = 2;
+ int iterationsLeft = 3;
Set<ContainerId> releases = new TreeSet<ContainerId>();
NMTokenCache.clearCache();
@@ -532,7 +566,7 @@ public class TestAMRMClient {
if(allocatedContainerCount < containersRequestedAny) {
// sleep to let NM's heartbeat to RM and trigger allocations
- sleep(1000);
+ sleep(100);
}
}
@@ -546,7 +580,9 @@ public class TestAMRMClient {
// need to tell the AMRMClient that we dont need these resources anymore
amClient.removeContainerRequest(
- new ContainerRequest(capability, nodes, racks, priority, 2));
+ new ContainerRequest(capability, nodes, racks, priority));
+ amClient.removeContainerRequest(
+ new ContainerRequest(capability, nodes, racks, priority));
assertTrue(amClient.ask.size() == 3);
// send 0 container count request for resources that are no longer needed
ResourceRequest snoopRequest = amClient.ask.iterator().next();
@@ -554,7 +590,9 @@ public class TestAMRMClient {
// test RPC exception handling
amClient.addContainerRequest(new ContainerRequest(capability, nodes,
- racks, priority, 2));
+ racks, priority));
+ amClient.addContainerRequest(new ContainerRequest(capability, nodes,
+ racks, priority));
snoopRequest = amClient.ask.iterator().next();
assertTrue(snoopRequest.getNumContainers() == 2);
@@ -567,7 +605,9 @@ public class TestAMRMClient {
throws Exception {
amClient.removeContainerRequest(
new ContainerRequest(capability, nodes,
- racks, priority, 2));
+ racks, priority));
+ amClient.removeContainerRequest(
+ new ContainerRequest(capability, nodes, racks, priority));
throw new Exception();
}
});
@@ -585,7 +625,7 @@ public class TestAMRMClient {
// has not been lost
assertTrue(snoopRequest.getNumContainers() == 0);
- iterationsLeft = 2;
+ iterationsLeft = 3;
// do a few iterations to ensure RM is not going send new containers
while(!releases.isEmpty() || iterationsLeft-- > 0) {
// inform RM of rejection
@@ -604,7 +644,7 @@ public class TestAMRMClient {
}
if(iterationsLeft > 0) {
// sleep to make sure NM's heartbeat
- sleep(1000);
+ sleep(100);
}
}
assertTrue(amClient.ask.size() == 0);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java Mon Aug 12 21:25:49 2013
@@ -18,31 +18,27 @@
package org.apache.hadoop.yarn.client.api.impl;
+import static org.junit.Assert.assertEquals;
+
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.net.DNSToSwitchMapping;
-import org.apache.hadoop.net.NetworkTopology;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
-import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
public class TestAMRMClientContainerRequest {
@Test
public void testFillInRacks() {
- AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>(
- ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0));
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
Configuration conf = new Configuration();
conf.setClass(
@@ -53,7 +49,7 @@ public class TestAMRMClientContainerRequ
Resource capability = Resource.newInstance(1024, 1);
ContainerRequest request =
new ContainerRequest(capability, new String[] {"host1", "host2"},
- new String[] {"/rack2"}, Priority.newInstance(1), 4);
+ new String[] {"/rack2"}, Priority.newInstance(1));
client.addContainerRequest(request);
verifyResourceRequest(client, request, "host1", true);
verifyResourceRequest(client, request, "host2", true);
@@ -64,8 +60,8 @@ public class TestAMRMClientContainerRequ
@Test
public void testDisableLocalityRelaxation() {
- AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>(
- ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0));
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
Configuration conf = new Configuration();
conf.setClass(
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
@@ -75,7 +71,7 @@ public class TestAMRMClientContainerRequ
Resource capability = Resource.newInstance(1024, 1);
ContainerRequest nodeLevelRequest =
new ContainerRequest(capability, new String[] {"host1", "host2"},
- null, Priority.newInstance(1), 4, false);
+ null, Priority.newInstance(1), false);
client.addContainerRequest(nodeLevelRequest);
verifyResourceRequest(client, nodeLevelRequest, ResourceRequest.ANY, false);
@@ -87,12 +83,12 @@ public class TestAMRMClientContainerRequ
// same priority
ContainerRequest nodeLevelRequest2 =
new ContainerRequest(capability, new String[] {"host2", "host3"},
- null, Priority.newInstance(1), 4, false);
+ null, Priority.newInstance(1), false);
client.addContainerRequest(nodeLevelRequest2);
AMRMClient.ContainerRequest rackLevelRequest =
new AMRMClient.ContainerRequest(capability, null,
- new String[] {"/rack3", "/rack4"}, Priority.newInstance(2), 3, false);
+ new String[] {"/rack3", "/rack4"}, Priority.newInstance(2), false);
client.addContainerRequest(rackLevelRequest);
verifyResourceRequest(client, rackLevelRequest, ResourceRequest.ANY, false);
@@ -103,13 +99,13 @@ public class TestAMRMClientContainerRequ
// same priority
AMRMClient.ContainerRequest rackLevelRequest2 =
new AMRMClient.ContainerRequest(capability, null,
- new String[] {"/rack4", "/rack5"}, Priority.newInstance(2), 3, false);
+ new String[] {"/rack4", "/rack5"}, Priority.newInstance(2), false);
client.addContainerRequest(rackLevelRequest2);
ContainerRequest bothLevelRequest =
new ContainerRequest(capability, new String[] {"host3", "host4"},
new String[] {"rack1", "/otherrack"},
- Priority.newInstance(3), 4, false);
+ Priority.newInstance(3), false);
client.addContainerRequest(bothLevelRequest);
verifyResourceRequest(client, bothLevelRequest, ResourceRequest.ANY, false);
@@ -125,14 +121,14 @@ public class TestAMRMClientContainerRequ
ContainerRequest bothLevelRequest2 =
new ContainerRequest(capability, new String[] {"host4", "host5"},
new String[] {"rack1", "/otherrack2"},
- Priority.newInstance(3), 4, false);
+ Priority.newInstance(3), false);
client.addContainerRequest(bothLevelRequest2);
}
@Test (expected = InvalidContainerRequestException.class)
public void testDifferentLocalityRelaxationSamePriority() {
- AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>(
- ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0));
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
Configuration conf = new Configuration();
conf.setClass(
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
@@ -142,18 +138,18 @@ public class TestAMRMClientContainerRequ
Resource capability = Resource.newInstance(1024, 1);
ContainerRequest request1 =
new ContainerRequest(capability, new String[] {"host1", "host2"},
- null, Priority.newInstance(1), 4, false);
+ null, Priority.newInstance(1), false);
client.addContainerRequest(request1);
ContainerRequest request2 =
new ContainerRequest(capability, new String[] {"host3"},
- null, Priority.newInstance(1), 4, true);
+ null, Priority.newInstance(1), true);
client.addContainerRequest(request2);
}
@Test
public void testInvalidValidWhenOldRemoved() {
- AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>(
- ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0));
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
Configuration conf = new Configuration();
conf.setClass(
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
@@ -163,36 +159,36 @@ public class TestAMRMClientContainerRequ
Resource capability = Resource.newInstance(1024, 1);
ContainerRequest request1 =
new ContainerRequest(capability, new String[] {"host1", "host2"},
- null, Priority.newInstance(1), 4, false);
+ null, Priority.newInstance(1), false);
client.addContainerRequest(request1);
client.removeContainerRequest(request1);
ContainerRequest request2 =
new ContainerRequest(capability, new String[] {"host3"},
- null, Priority.newInstance(1), 4, true);
+ null, Priority.newInstance(1), true);
client.addContainerRequest(request2);
client.removeContainerRequest(request2);
ContainerRequest request3 =
new ContainerRequest(capability, new String[] {"host1", "host2"},
- null, Priority.newInstance(1), 4, false);
+ null, Priority.newInstance(1), false);
client.addContainerRequest(request3);
client.removeContainerRequest(request3);
ContainerRequest request4 =
new ContainerRequest(capability, null,
- new String[] {"rack1"}, Priority.newInstance(1), 4, true);
+ new String[] {"rack1"}, Priority.newInstance(1), true);
client.addContainerRequest(request4);
}
@Test (expected = InvalidContainerRequestException.class)
public void testLocalityRelaxationDifferentLevels() {
- AMRMClientImpl<ContainerRequest> client = new AMRMClientImpl<ContainerRequest>(
- ApplicationAttemptId.newInstance(ApplicationId.newInstance(0l, 0), 0));
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
Configuration conf = new Configuration();
conf.setClass(
CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
@@ -202,11 +198,11 @@ public class TestAMRMClientContainerRequ
Resource capability = Resource.newInstance(1024, 1);
ContainerRequest request1 =
new ContainerRequest(capability, new String[] {"host1", "host2"},
- null, Priority.newInstance(1), 4, false);
+ null, Priority.newInstance(1), false);
client.addContainerRequest(request1);
ContainerRequest request2 =
new ContainerRequest(capability, null,
- new String[] {"rack1"}, Priority.newInstance(1), 4, true);
+ new String[] {"rack1"}, Priority.newInstance(1), true);
client.addContainerRequest(request2);
}
@@ -227,7 +223,7 @@ public class TestAMRMClientContainerRequ
ResourceRequest ask = client.remoteRequestsTable.get(request.getPriority())
.get(location).get(request.getCapability()).remoteRequest;
assertEquals(location, ask.getResourceName());
- assertEquals(request.getContainerCount(), ask.getNumContainers());
+ assertEquals(1, ask.getNumContainers());
assertEquals(expectedRelaxLocality, ask.getRelaxLocality());
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Mon Aug 12 21:25:49 2013
@@ -26,14 +26,15 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
@@ -52,16 +53,17 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.Before;
@@ -124,11 +126,20 @@ public class TestNMClient {
// wait for app to start
int iterationsLeft = 30;
+ RMAppAttempt appAttempt = null;
while (iterationsLeft > 0) {
ApplicationReport appReport = yarnClient.getApplicationReport(appId);
if (appReport.getYarnApplicationState() ==
YarnApplicationState.ACCEPTED) {
attemptId = appReport.getCurrentApplicationAttemptId();
+ appAttempt =
+ yarnCluster.getResourceManager().getRMContext().getRMApps()
+ .get(attemptId.getApplicationId()).getCurrentAppAttempt();
+ while (true) {
+ if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
+ break;
+ }
+ }
break;
}
sleep(1000);
@@ -138,10 +149,16 @@ public class TestNMClient {
fail("Application hasn't bee started");
}
+ // Just dig into the ResourceManager and get the AMRMToken just for the sake
+ // of testing.
+ UserGroupInformation.setLoginUser(UserGroupInformation
+ .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
+ UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
+
// start am rm client
rmClient =
(AMRMClientImpl<ContainerRequest>) AMRMClient
- .<ContainerRequest> createAMRMClient(attemptId);
+ .<ContainerRequest> createAMRMClient();
rmClient.init(conf);
rmClient.start();
assertNotNull(rmClient);
@@ -220,7 +237,7 @@ public class TestNMClient {
for (int i = 0; i < num; ++i) {
rmClient.addContainerRequest(new ContainerRequest(capability, nodes,
- racks, priority, 1));
+ racks, priority));
}
int containersRequestedAny = rmClient.remoteRequestsTable.get(priority)
@@ -303,7 +320,7 @@ public class TestNMClient {
if (++i < size) {
// NodeManager may still need some time to make the container started
testGetContainerStatus(container, i, ContainerState.RUNNING, "",
- -1000);
+ Arrays.asList(new Integer[] {-1000}));
try {
nmClient.stopContainer(container.getId(), container.getNodeId());
@@ -314,8 +331,21 @@ public class TestNMClient {
}
// getContainerStatus can be called after stopContainer
- testGetContainerStatus(container, i, ContainerState.COMPLETE,
- "Container killed by the ApplicationMaster.", 143);
+ try {
+ // O is possible if CLEANUP_CONTAINER is executed too late
+ testGetContainerStatus(container, i, ContainerState.COMPLETE,
+ "Container killed by the ApplicationMaster.", Arrays.asList(
+ new Integer[] {143, 0}));
+ } catch (YarnException e) {
+ // The exception is possible because, after the container is stopped,
+ // it may be removed from NM's context.
+ if (!e.getMessage()
+ .contains("was recently stopped on node manager")) {
+ throw (AssertionError)
+ (new AssertionError("Exception is not expected: " + e).initCause(
+ e));
+ }
+ }
}
}
}
@@ -329,7 +359,7 @@ public class TestNMClient {
}
private void testGetContainerStatus(Container container, int index,
- ContainerState state, String diagnostics, int exitStatus)
+ ContainerState state, String diagnostics, List<Integer> exitStatuses)
throws YarnException, IOException {
while (true) {
try {
@@ -341,7 +371,7 @@ public class TestNMClient {
assertEquals(container.getId(), status.getContainerId());
assertTrue("" + index + ": " + status.getDiagnostics(),
status.getDiagnostics().contains(diagnostics));
- assertEquals(exitStatus, status.getExitStatus());
+ assertTrue(exitStatuses.contains(status.getExitStatus()));
break;
}
Thread.sleep(100);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Mon Aug 12 21:25:49 2013
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.verify
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -34,6 +35,7 @@ import java.util.Set;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
@@ -44,15 +46,20 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
-import org.apache.hadoop.yarn.client.api.impl.YarnClientImpl;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -237,7 +244,7 @@ public class TestYarnClient {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
List<ApplicationReport> applicationReports =
new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
@@ -247,7 +254,8 @@ public class TestYarnClient {
applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2),
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
+ null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -255,7 +263,8 @@ public class TestYarnClient {
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
+ null);
applicationReports.add(newApplicationReport3);
return applicationReports;
}
@@ -281,4 +290,107 @@ public class TestYarnClient {
}
}
+ @Test(timeout = 30000)
+ public void testAMMRTokens() throws Exception {
+ MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
+ YarnClient rmClient = null;
+ try {
+ cluster.init(new YarnConfiguration());
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
+ rmClient = YarnClient.createYarnClient();
+ rmClient.init(yarnConf);
+ rmClient.start();
+
+ ApplicationId appId = createApp(rmClient, false);
+ waitTillAccepted(rmClient, appId);
+ //managed AMs don't return AMRM token
+ Assert.assertNull(rmClient.getAMRMToken(appId));
+
+ appId = createApp(rmClient, true);
+ waitTillAccepted(rmClient, appId);
+ //unmanaged AMs do return AMRM token
+ Assert.assertNotNull(rmClient.getAMRMToken(appId));
+
+ UserGroupInformation other =
+ UserGroupInformation.createUserForTesting("foo", new String[]{});
+ appId = other.doAs(
+ new PrivilegedExceptionAction<ApplicationId>() {
+ @Override
+ public ApplicationId run() throws Exception {
+ YarnClient rmClient = YarnClient.createYarnClient();
+ rmClient.init(yarnConf);
+ rmClient.start();
+ ApplicationId appId = createApp(rmClient, true);
+ waitTillAccepted(rmClient, appId);
+ //unmanaged AMs do return AMRM token
+ Assert.assertNotNull(rmClient.getAMRMToken(appId));
+ return appId;
+ }
+ });
+ //other users don't get AMRM token
+ Assert.assertNull(rmClient.getAMRMToken(appId));
+ } finally {
+ if (rmClient != null) {
+ rmClient.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ private ApplicationId createApp(YarnClient rmClient, boolean unmanaged)
+ throws Exception {
+ YarnClientApplication newApp = rmClient.createApplication();
+
+ ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();
+
+ // Create launch context for app master
+ ApplicationSubmissionContext appContext
+ = Records.newRecord(ApplicationSubmissionContext.class);
+
+ // set the application id
+ appContext.setApplicationId(appId);
+
+ // set the application name
+ appContext.setApplicationName("test");
+
+ // Set the priority for the application master
+ Priority pri = Records.newRecord(Priority.class);
+ pri.setPriority(1);
+ appContext.setPriority(pri);
+
+ // Set the queue to which this application is to be submitted in the RM
+ appContext.setQueue("default");
+
+ // Set up the container launch context for the application master
+ ContainerLaunchContext amContainer
+ = Records.newRecord(ContainerLaunchContext.class);
+ appContext.setAMContainerSpec(amContainer);
+ appContext.setResource(Resource.newInstance(1024, 1));
+ appContext.setUnmanagedAM(unmanaged);
+
+ // Submit the application to the applications manager
+ rmClient.submitApplication(appContext);
+
+ return appId;
+ }
+
+ private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
+ throws Exception {
+ try {
+ long start = System.currentTimeMillis();
+ ApplicationReport report = rmClient.getApplicationReport(appId);
+ while (YarnApplicationState.ACCEPTED != report.getYarnApplicationState()) {
+ if (System.currentTimeMillis() - start > 20 * 1000) {
+ throw new Exception("App '" + appId +
+ "' time out, failed to reach ACCEPTED state");
+ }
+ Thread.sleep(200);
+ report = rmClient.getApplicationReport(appId);
+ }
+ } catch (Exception ex) {
+ throw new Exception(ex);
+ }
+ }
+
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java Mon Aug 12 21:25:49 2013
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Before;
import org.junit.Test;
@@ -79,7 +80,7 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
newApplicationReport);
int result = cli.run(new String[] { "-status", applicationId.toString() });
@@ -109,6 +110,23 @@ public class TestYarnCLI {
}
@Test
+ public void testGetApplicationReportException() throws Exception {
+ ApplicationCLI cli = createAndGetAppCLI();
+ ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
+ when(client.getApplicationReport(any(ApplicationId.class))).thenThrow(
+ new ApplicationNotFoundException("Application with id '"
+ + applicationId + "' doesn't exist in RM."));
+ try {
+ cli.run(new String[] { "-status", applicationId.toString() });
+ Assert.fail();
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof ApplicationNotFoundException);
+ Assert.assertEquals("Application with id '" + applicationId
+ + "' doesn't exist in RM.", ex.getMessage());
+ }
+ }
+
+ @Test
public void testGetApplications() throws Exception {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
@@ -116,7 +134,7 @@ public class TestYarnCLI {
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
applicationReports.add(newApplicationReport);
@@ -125,7 +143,8 @@ public class TestYarnCLI {
applicationId2, ApplicationAttemptId.newInstance(applicationId2, 2),
"user2", "queue2", "appname2", "host2", 125, null,
YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN",
+ null);
applicationReports.add(newApplicationReport2);
ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -133,7 +152,8 @@ public class TestYarnCLI {
applicationId3, ApplicationAttemptId.newInstance(applicationId3, 3),
"user3", "queue3", "appname3", "host3", 126, null,
YarnApplicationState.FINISHED, "diagnostics3", "url3", 3, 3,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE");
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
+ null);
applicationReports.add(newApplicationReport3);
Set<String> appType1 = new HashSet<String>();
@@ -355,8 +375,10 @@ public class TestYarnCLI {
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0M");
- pw.println("\tMemory-Capacity : 0");
+ pw.println("\tMemory-Used : 0MB");
+ pw.println("\tMemory-Capacity : 0MB");
+ pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tCPU-Capacity : 0 vcores");
pw.close();
String nodeStatusStr = baos.toString("UTF-8");
verify(sysOut, times(1)).println(isA(String.class));
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Mon Aug 12 21:25:49 2013
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<name>hadoop-yarn-common</name>
<properties>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/client/ContainerManagementProtocolPBClientImpl.java Mon Aug 12 21:25:49 2013
@@ -30,24 +30,24 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusResponsePBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerResponsePBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto;
import com.google.protobuf.ServiceException;
@@ -87,14 +87,14 @@ public class ContainerManagementProtocol
}
@Override
- public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException,
- IOException {
- GetContainerStatusRequestProto requestProto =
- ((GetContainerStatusRequestPBImpl) request).getProto();
+ public StartContainersResponse
+ startContainers(StartContainersRequest requests) throws YarnException,
+ IOException {
+ StartContainersRequestProto requestProto =
+ ((StartContainersRequestPBImpl) requests).getProto();
try {
- return new GetContainerStatusResponsePBImpl(proxy.getContainerStatus(
- null, requestProto));
+ return new StartContainersResponsePBImpl(proxy.startContainers(null,
+ requestProto));
} catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e);
return null;
@@ -102,12 +102,12 @@ public class ContainerManagementProtocol
}
@Override
- public StartContainerResponse startContainer(StartContainerRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest requests)
throws YarnException, IOException {
- StartContainerRequestProto requestProto =
- ((StartContainerRequestPBImpl) request).getProto();
+ StopContainersRequestProto requestProto =
+ ((StopContainersRequestPBImpl) requests).getProto();
try {
- return new StartContainerResponsePBImpl(proxy.startContainer(null,
+ return new StopContainersResponsePBImpl(proxy.stopContainers(null,
requestProto));
} catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e);
@@ -116,13 +116,13 @@ public class ContainerManagementProtocol
}
@Override
- public StopContainerResponse stopContainer(StopContainerRequest request)
- throws YarnException, IOException {
- StopContainerRequestProto requestProto =
- ((StopContainerRequestPBImpl) request).getProto();
+ public GetContainerStatusesResponse getContainerStatuses(
+ GetContainerStatusesRequest request) throws YarnException, IOException {
+ GetContainerStatusesRequestProto requestProto =
+ ((GetContainerStatusesRequestPBImpl) request).getProto();
try {
- return new StopContainerResponsePBImpl(proxy.stopContainer(null,
- requestProto));
+ return new GetContainerStatusesResponsePBImpl(proxy.getContainerStatuses(
+ null, requestProto));
} catch (ServiceException e) {
RPCUtil.unwrapAndThrowException(e);
return null;
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/impl/pb/service/ContainerManagementProtocolPBServiceImpl.java Mon Aug 12 21:25:49 2013
@@ -23,22 +23,22 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.ContainerManagementProtocolPB;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusResponsePBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainerResponsePBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerRequestPBImpl;
-import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainerResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.GetContainerStatusesResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StartContainersResponsePBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestPBImpl;
+import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusResponseProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerResponseProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainerResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetContainerStatusesResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainersResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -53,12 +53,12 @@ public class ContainerManagementProtocol
}
@Override
- public GetContainerStatusResponseProto getContainerStatus(RpcController arg0,
- GetContainerStatusRequestProto proto) throws ServiceException {
- GetContainerStatusRequestPBImpl request = new GetContainerStatusRequestPBImpl(proto);
+ public StartContainersResponseProto startContainers(RpcController arg0,
+ StartContainersRequestProto proto) throws ServiceException {
+ StartContainersRequestPBImpl request = new StartContainersRequestPBImpl(proto);
try {
- GetContainerStatusResponse response = real.getContainerStatus(request);
- return ((GetContainerStatusResponsePBImpl)response).getProto();
+ StartContainersResponse response = real.startContainers(request);
+ return ((StartContainersResponsePBImpl)response).getProto();
} catch (YarnException e) {
throw new ServiceException(e);
} catch (IOException e) {
@@ -67,12 +67,12 @@ public class ContainerManagementProtocol
}
@Override
- public StartContainerResponseProto startContainer(RpcController arg0,
- StartContainerRequestProto proto) throws ServiceException {
- StartContainerRequestPBImpl request = new StartContainerRequestPBImpl(proto);
+ public StopContainersResponseProto stopContainers(RpcController arg0,
+ StopContainersRequestProto proto) throws ServiceException {
+ StopContainersRequestPBImpl request = new StopContainersRequestPBImpl(proto);
try {
- StartContainerResponse response = real.startContainer(request);
- return ((StartContainerResponsePBImpl)response).getProto();
+ StopContainersResponse response = real.stopContainers(request);
+ return ((StopContainersResponsePBImpl)response).getProto();
} catch (YarnException e) {
throw new ServiceException(e);
} catch (IOException e) {
@@ -81,17 +81,17 @@ public class ContainerManagementProtocol
}
@Override
- public StopContainerResponseProto stopContainer(RpcController arg0,
- StopContainerRequestProto proto) throws ServiceException {
- StopContainerRequestPBImpl request = new StopContainerRequestPBImpl(proto);
+ public GetContainerStatusesResponseProto getContainerStatuses(
+ RpcController arg0, GetContainerStatusesRequestProto proto)
+ throws ServiceException {
+ GetContainerStatusesRequestPBImpl request = new GetContainerStatusesRequestPBImpl(proto);
try {
- StopContainerResponse response = real.stopContainer(request);
- return ((StopContainerResponsePBImpl)response).getProto();
+ GetContainerStatusesResponse response = real.getContainerStatuses(request);
+ return ((GetContainerStatusesResponsePBImpl)response).getProto();
} catch (YarnException e) {
throw new ServiceException(e);
} catch (IOException e) {
throw new ServiceException(e);
}
}
-
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java Mon Aug 12 21:25:49 2013
@@ -26,15 +26,12 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
@@ -48,7 +45,6 @@ public class AllocateRequestPBImpl exten
AllocateRequestProto.Builder builder = null;
boolean viaProto = false;
- private ApplicationAttemptId applicationAttemptID = null;
private List<ResourceRequest> ask = null;
private List<ContainerId> release = null;
private ResourceBlacklistRequest blacklistRequest = null;
@@ -91,9 +87,6 @@ public class AllocateRequestPBImpl exten
}
private void mergeLocalToBuilder() {
- if (this.applicationAttemptID != null) {
- builder.setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptID));
- }
if (this.ask != null) {
addAsksToProto();
}
@@ -121,27 +114,6 @@ public class AllocateRequestPBImpl exten
}
@Override
- public ApplicationAttemptId getApplicationAttemptId() {
- AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (this.applicationAttemptID != null) {
- return this.applicationAttemptID;
- }
- if (!p.hasApplicationAttemptId()) {
- return null;
- }
- this.applicationAttemptID = convertFromProtoFormat(p.getApplicationAttemptId());
- return this.applicationAttemptID;
- }
-
- @Override
- public void setApplicationAttemptId(ApplicationAttemptId appAttemptId) {
- maybeInitBuilder();
- if (appAttemptId == null)
- builder.clearApplicationAttemptId();
- this.applicationAttemptID = appAttemptId;
- }
-
- @Override
public int getResponseId() {
AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
return p.getResponseId();
@@ -311,14 +283,6 @@ public class AllocateRequestPBImpl exten
builder.addAllRelease(iterable);
}
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
- return new ApplicationAttemptIdPBImpl(p);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) {
- return ((ApplicationAttemptIdPBImpl)t).getProto();
- }
-
private ResourceRequestPBImpl convertFromProtoFormat(ResourceRequestProto p) {
return new ResourceRequestPBImpl(p);
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/FinishApplicationMasterRequestPBImpl.java Mon Aug 12 21:25:49 2013
@@ -22,11 +22,8 @@ package org.apache.hadoop.yarn.api.proto
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.FinalApplicationStatusProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProtoOrBuilder;
@@ -38,9 +35,6 @@ public class FinishApplicationMasterRequ
FinishApplicationMasterRequestProto.Builder builder = null;
boolean viaProto = false;
- private ApplicationAttemptId appAttemptId = null;
-
-
public FinishApplicationMasterRequestPBImpl() {
builder = FinishApplicationMasterRequestProto.newBuilder();
}
@@ -78,9 +72,6 @@ public class FinishApplicationMasterRequ
}
private void mergeLocalToBuilder() {
- if (this.appAttemptId != null) {
- builder.setApplicationAttemptId(convertToProtoFormat(this.appAttemptId));
- }
}
private void mergeLocalToProto() {
@@ -99,27 +90,6 @@ public class FinishApplicationMasterRequ
}
@Override
- public ApplicationAttemptId getApplicationAttemptId() {
- FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (this.appAttemptId != null) {
- return this.appAttemptId;
- }
- if (!p.hasApplicationAttemptId()) {
- return null;
- }
- this.appAttemptId = convertFromProtoFormat(p.getApplicationAttemptId());
- return this.appAttemptId;
- }
-
- @Override
- public void setAppAttemptId(ApplicationAttemptId applicationAttemptId) {
- maybeInitBuilder();
- if (applicationAttemptId == null)
- builder.clearApplicationAttemptId();
- this.appAttemptId = applicationAttemptId;
- }
-
- @Override
public String getDiagnostics() {
FinishApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
return p.getDiagnostics();
@@ -128,6 +98,10 @@ public class FinishApplicationMasterRequ
@Override
public void setDiagnostics(String diagnostics) {
maybeInitBuilder();
+ if (diagnostics == null) {
+ builder.clearDiagnostics();
+ return;
+ }
builder.setDiagnostics(diagnostics);
}
@@ -140,6 +114,10 @@ public class FinishApplicationMasterRequ
@Override
public void setTrackingUrl(String url) {
maybeInitBuilder();
+ if (url == null) {
+ builder.clearTrackingUrl();
+ return;
+ }
builder.setTrackingUrl(url);
}
@@ -162,14 +140,6 @@ public class FinishApplicationMasterRequ
builder.setFinalApplicationStatus(convertToProtoFormat(finalState));
}
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
- return new ApplicationAttemptIdPBImpl(p);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) {
- return ((ApplicationAttemptIdPBImpl)t).getProto();
- }
-
private FinalApplicationStatus convertFromProtoFormat(FinalApplicationStatusProto s) {
return ProtoUtils.convertFromProtoFormat(s);
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java Mon Aug 12 21:25:49 2013
@@ -22,9 +22,6 @@ package org.apache.hadoop.yarn.api.proto
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAttemptIdProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProtoOrBuilder;
@@ -35,9 +32,6 @@ public class RegisterApplicationMasterRe
RegisterApplicationMasterRequestProto.Builder builder = null;
boolean viaProto = false;
- private ApplicationAttemptId applicationAttemptId = null;
-
-
public RegisterApplicationMasterRequestPBImpl() {
builder = RegisterApplicationMasterRequestProto.newBuilder();
}
@@ -75,9 +69,6 @@ public class RegisterApplicationMasterRe
}
private void mergeLocalToBuilder() {
- if (this.applicationAttemptId != null && !((ApplicationAttemptIdPBImpl)this.applicationAttemptId).getProto().equals(builder.getApplicationAttemptId())) {
- builder.setApplicationAttemptId(convertToProtoFormat(this.applicationAttemptId));
- }
}
private void mergeLocalToProto() {
@@ -94,28 +85,6 @@ public class RegisterApplicationMasterRe
}
viaProto = false;
}
-
-
- @Override
- public ApplicationAttemptId getApplicationAttemptId() {
- RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (this.applicationAttemptId != null) {
- return this.applicationAttemptId;
- }
- if (!p.hasApplicationAttemptId()) {
- return null;
- }
- this.applicationAttemptId = convertFromProtoFormat(p.getApplicationAttemptId());
- return this.applicationAttemptId;
- }
-
- @Override
- public void setApplicationAttemptId(ApplicationAttemptId applicationMaster) {
- maybeInitBuilder();
- if (applicationMaster == null)
- builder.clearApplicationAttemptId();
- this.applicationAttemptId = applicationMaster;
- }
@Override
public String getHost() {
@@ -126,6 +95,10 @@ public class RegisterApplicationMasterRe
@Override
public void setHost(String host) {
maybeInitBuilder();
+ if (host == null) {
+ builder.clearHost();
+ return;
+ }
builder.setHost(host);
}
@@ -150,15 +123,10 @@ public class RegisterApplicationMasterRe
@Override
public void setTrackingUrl(String url) {
maybeInitBuilder();
+ if (url == null) {
+ builder.clearTrackingUrl();
+ return;
+ }
builder.setTrackingUrl(url);
}
-
- private ApplicationAttemptIdPBImpl convertFromProtoFormat(ApplicationAttemptIdProto p) {
- return new ApplicationAttemptIdPBImpl(p);
- }
-
- private ApplicationAttemptIdProto convertToProtoFormat(ApplicationAttemptId t) {
- return ((ApplicationAttemptIdPBImpl)t).getProto();
- }
-
}