You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/08/12 23:26:19 UTC
svn commit: r1513258 [10/10] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/conf/ hadoop-yarn/dev-support/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Mon Aug 12 21:25:49 2013
@@ -638,6 +638,191 @@ public class TestRMWebServicesApps exten
}
@Test
+ public void testAppsQueryAppTypes() throws JSONException, Exception {
+ rm.start();
+ MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
+ Thread.sleep(1);
+ RMApp app1 = rm.submitApp(1024);
+ amNodeManager.nodeHeartbeat(true);
+ // finish App
+ MockAM am = rm
+ .sendAMLaunched(app1.getCurrentAppAttempt().getAppAttemptId());
+ am.registerAppAttempt();
+ am.unregisterAppAttempt();
+ amNodeManager.nodeHeartbeat(app1.getCurrentAppAttempt().getAppAttemptId(),
+ 1, ContainerState.COMPLETE);
+
+ rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "MAPREDUCE");
+ rm.submitApp(1024, "", UserGroupInformation.getCurrentUser()
+ .getShortUserName(), null, false, null, 2, null, "NON-YARN");
+
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ JSONArray array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ assertEquals("MAPREDUCE",
+ array.getJSONObject(0).getString("applicationType"));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN")
+ .queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("MAPREDUCE")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("MAPREDUCE")));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN,NON-YARN")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("NON-YARN")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("NON-YARN")));
+
+ r = resource();
+ response = r.path("ws").path("v1").path("cluster")
+ .path("apps").queryParam("applicationTypes", "")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 3, array.length());
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN,NON-YARN")
+ .queryParam("applicationTypes", "MAPREDUCE")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 3, array.length());
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN")
+ .queryParam("applicationTypes", "")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ assertEquals("YARN",
+ array.getJSONObject(0).getString("applicationType"));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", ",,, ,, YARN ,, ,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 1, array.length());
+ assertEquals("YARN",
+ array.getJSONObject(0).getString("applicationType"));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", ",,, ,, ,, ,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 3, array.length());
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", "YARN, ,NON-YARN, ,,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("NON-YARN")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("NON-YARN")));
+
+ r = resource();
+ response =
+ r.path("ws").path("v1").path("cluster").path("apps")
+ .queryParam("applicationTypes", " YARN, , ,,,")
+ .queryParam("applicationTypes", "MAPREDUCE , ,, ,")
+ .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ apps = json.getJSONObject("apps");
+ assertEquals("incorrect number of elements", 1, apps.length());
+ array = apps.getJSONArray("app");
+ assertEquals("incorrect number of elements", 2, array.length());
+ assertTrue((array.getJSONObject(0).getString("applicationType")
+ .equals("YARN") && array.getJSONObject(1).getString("applicationType")
+ .equals("MAPREDUCE")) ||
+ (array.getJSONObject(1).getString("applicationType").equals("YARN")
+ && array.getJSONObject(0).getString("applicationType")
+ .equals("MAPREDUCE")));
+
+ rm.stop();
+ }
+
+ @Test
public void testSingleApp() throws JSONException, Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Mon Aug 12 21:25:49 2013
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<name>hadoop-yarn-server-tests</name>
<properties>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Mon Aug 12 21:25:49 2013
@@ -304,6 +304,16 @@ public class MiniYARNCluster extends Com
MiniYARNCluster.getHostname() + ":0");
getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS,
MiniYARNCluster.getHostname() + ":0");
+
+ // Disable resource checks by default
+ if (!getConfig().getBoolean(
+ YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
+ YarnConfiguration.
+ DEFAULT_YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
+ getConfig().setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
+ getConfig().setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
+ }
+
LOG.info("Starting NM: " + index);
nodeManagers[index].init(getConfig());
new Thread() {
@@ -390,6 +400,11 @@ public class MiniYARNCluster extends Com
}
};
};
+
+ @Override
+ protected void stopRMProxy() {
+ return;
+ }
};
};
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Mon Aug 12 21:25:49 2013
@@ -23,6 +23,10 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
import junit.framework.Assert;
@@ -32,22 +36,29 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
-import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -56,7 +67,11 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+@RunWith(Parameterized.class)
public class TestContainerManagerSecurity {
static Log LOG = LogFactory.getLog(TestContainerManagerSecurity.class);
@@ -64,28 +79,33 @@ public class TestContainerManagerSecurit
.getRecordFactory(null);
private static MiniYARNCluster yarnCluster;
- static final Configuration conf = new Configuration();
+ private Configuration conf;
- @Test (timeout = 1000000)
- public void testContainerManagerWithSecurityEnabled() throws Exception {
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ @Parameters
+ public static Collection<Object[]> configs() {
+ Configuration configurationWithoutSecurity = new Configuration();
+ configurationWithoutSecurity.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "simple");
+
+ Configuration configurationWithSecurity = new Configuration();
+ configurationWithSecurity.set(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
- testContainerManager();
+ return Arrays.asList(new Object[][] { { configurationWithoutSecurity },
+ { configurationWithSecurity } });
}
- @Test (timeout=1000000)
- public void testContainerManagerWithSecurityDisabled() throws Exception {
- conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- testContainerManager();
+ public TestContainerManagerSecurity(Configuration conf) {
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
+ UserGroupInformation.setConfiguration(conf);
+ this.conf = conf;
}
- private void testContainerManager() throws Exception {
+ @Test (timeout = 1000000)
+ public void testContainerManager() throws Exception {
try {
yarnCluster = new MiniYARNCluster(TestContainerManagerSecurity.class
.getName(), 1, 1, 1);
- conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 100000L);
- UserGroupInformation.setConfiguration(conf);
yarnCluster.init(conf);
yarnCluster.start();
@@ -177,6 +197,18 @@ public class TestContainerManagerSecurit
} while (tempManager.getCurrentKey().getKeyId() == nmTokenSecretManagerRM
.getCurrentKey().getKeyId());
+ // Testing that NM rejects the requests when we don't send any token.
+ if (UserGroupInformation.isSecurityEnabled()) {
+ sb = new StringBuilder("Client cannot authenticate via:[TOKEN]");
+ } else {
+ sb =
+ new StringBuilder(
+ "SIMPLE authentication is not enabled. Available:[TOKEN]");
+ }
+ String errorMsg = testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, null, true);
+ Assert.assertTrue(errorMsg.contains(sb.toString()));
+
org.apache.hadoop.yarn.api.records.Token invalidNMToken =
tempManager.createNMToken(validAppAttemptId, validNode, user);
sb = new StringBuilder("Given NMToken for application : ");
@@ -211,12 +243,30 @@ public class TestContainerManagerSecurit
Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
validContainerToken, invalidNMToken, true).contains(sb.toString()));
- // using correct tokens. nmtoken for appattempt should get saved.
+ // using correct tokens. nmtoken for app attempt should get saved.
+ conf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+ 4 * 60 * 1000);
+ validContainerToken =
+ containerTokenSecretManager.createContainerToken(validContainerId,
+ validNode, user, r);
+
testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken,
validNMToken, false);
Assert.assertTrue(nmTokenSecretManagerNM
.isAppAttemptNMTokenKeyPresent(validAppAttemptId));
+ //Now lets wait till container finishes and is removed from node manager.
+ waitForContainerToFinishOnNM(validContainerId);
+ sb = new StringBuilder("Attempt to relaunch the same container with id ");
+ sb.append(validContainerId);
+ Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, validNMToken, true).contains(sb.toString()));
+
+ // Container is removed from node manager's memory by this time.
+ // trying to stop the container. It should not throw any exception.
+ testStopContainer(rpc, validAppAttemptId, validNode, validContainerId,
+ validNMToken, false);
+
// Rolling over master key twice so that we can check whether older keys
// are used for authentication.
rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
@@ -224,13 +274,38 @@ public class TestContainerManagerSecurit
rollNMTokenMasterKey(nmTokenSecretManagerRM, nmTokenSecretManagerNM);
// trying get container status. Now saved nmToken should be used for
- // authentication.
+ // authentication... It should complain saying container was recently
+ // stopped.
+ sb = new StringBuilder("Container ");
+ sb.append(validContainerId);
+ sb.append(" was recently stopped on node manager");
+ Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
+ validContainerId, validNMToken, true).contains(sb.toString()));
+
+ // Now lets remove the container from nm-memory
+ nm.getNodeStatusUpdater().clearFinishedContainersFromCache();
+
+ // This should fail as container is removed from recently tracked finished
+ // containers.
sb = new StringBuilder("Container ");
sb.append(validContainerId.toString());
sb.append(" is not handled by this NodeManager");
Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, false).contains(sb.toString()));
-
+
+ }
+
+ private void waitForContainerToFinishOnNM(ContainerId containerId) {
+ Context nmContet = yarnCluster.getNodeManager(0).getNMContext();
+ int interval = 4 * 60; // Max time for container token to expire.
+ while ((interval-- > 0)
+ && nmContet.getContainers().containsKey(containerId)) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ }
+ }
+ Assert.assertFalse(nmContet.getContainers().containsKey(containerId));
}
protected void waitForNMToReceiveNMTokenKey(
@@ -259,6 +334,23 @@ public class TestContainerManagerSecurit
Assert.assertTrue((nmTokenSecretManagerNM.getCurrentKey().getKeyId()
== nmTokenSecretManagerRM.getCurrentKey().getKeyId()));
}
+
+ private String testStopContainer(YarnRPC rpc,
+ ApplicationAttemptId appAttemptId, NodeId nodeId,
+ ContainerId containerId, Token nmToken, boolean isExceptionExpected) {
+ try {
+ stopContainer(rpc, nmToken,
+ Arrays.asList(new ContainerId[] { containerId }), appAttemptId,
+ nodeId);
+ if (isExceptionExpected) {
+ fail("Exception was expected!!");
+ }
+ return "";
+ } catch (Exception e) {
+ e.printStackTrace();
+ return e.getMessage();
+ }
+ }
private String testGetContainer(YarnRPC rpc,
ApplicationAttemptId appAttemptId, NodeId nodeId,
@@ -278,7 +370,7 @@ public class TestContainerManagerSecurit
}
}
- protected String testStartContainer(YarnRPC rpc,
+ private String testStartContainer(YarnRPC rpc,
ApplicationAttemptId appAttemptId, NodeId nodeId,
org.apache.hadoop.yarn.api.records.Token containerToken,
org.apache.hadoop.yarn.api.records.Token nmToken,
@@ -296,24 +388,50 @@ public class TestContainerManagerSecurit
}
}
+ private void stopContainer(YarnRPC rpc, Token nmToken,
+ List<ContainerId> containerId, ApplicationAttemptId appAttemptId,
+ NodeId nodeId) throws Exception {
+ StopContainersRequest request =
+ StopContainersRequest.newInstance(containerId);
+ ContainerManagementProtocol proxy = null;
+ try {
+ proxy =
+ getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
+ appAttemptId.toString());
+ StopContainersResponse response = proxy.stopContainers(request);
+ if (response.getFailedRequests() != null &&
+ response.getFailedRequests().containsKey(containerId)) {
+ parseAndThrowException(response.getFailedRequests().get(containerId)
+ .deSerialize());
+ }
+ } catch (Exception e) {
+ if (proxy != null) {
+ rpc.stopProxy(proxy, conf);
+ }
+ }
+ }
+
private void
getContainerStatus(YarnRPC rpc,
org.apache.hadoop.yarn.api.records.Token nmToken,
ContainerId containerId,
ApplicationAttemptId appAttemptId, NodeId nodeId,
boolean isExceptionExpected) throws Exception {
- GetContainerStatusRequest request =
- Records.newRecord(GetContainerStatusRequest.class);
- request.setContainerId(containerId);
-
+ List<ContainerId> containerIds = new ArrayList<ContainerId>();
+ containerIds.add(containerId);
+ GetContainerStatusesRequest request =
+ GetContainerStatusesRequest.newInstance(containerIds);
ContainerManagementProtocol proxy = null;
-
try {
proxy =
getContainerManagementProtocolProxy(rpc, nmToken, nodeId,
appAttemptId.toString());
- proxy.getContainerStatus(request);
-
+ GetContainerStatusesResponse statuses = proxy.getContainerStatuses(request);
+ if (statuses.getFailedRequests() != null
+ && statuses.getFailedRequests().containsKey(containerId)) {
+ parseAndThrowException(statuses.getFailedRequests().get(containerId)
+ .deSerialize());
+ }
} finally {
if (proxy != null) {
rpc.stopProxy(proxy, conf);
@@ -326,17 +444,21 @@ public class TestContainerManagerSecurit
org.apache.hadoop.yarn.api.records.Token containerToken,
NodeId nodeId, String user) throws Exception {
- StartContainerRequest request =
- Records.newRecord(StartContainerRequest.class);
- request.setContainerToken(containerToken);
ContainerLaunchContext context =
Records.newRecord(ContainerLaunchContext.class);
- request.setContainerLaunchContext(context);
-
+ StartContainerRequest scRequest =
+ StartContainerRequest.newInstance(context,containerToken);
+ List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
+ list.add(scRequest);
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
ContainerManagementProtocol proxy = null;
try {
proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
- proxy.startContainer(request);
+ StartContainersResponse response = proxy.startContainers(allRequests);
+ for(SerializedException ex : response.getFailedRequests().values()){
+ parseAndThrowException(ex.deSerialize());
+ }
} finally {
if (proxy != null) {
rpc.stopProxy(proxy, conf);
@@ -344,6 +466,17 @@ public class TestContainerManagerSecurit
}
}
+ private void parseAndThrowException(Throwable t) throws YarnException,
+ IOException {
+ if (t instanceof YarnException) {
+ throw (YarnException) t;
+ } else if (t instanceof InvalidToken) {
+ throw (InvalidToken) t;
+ } else {
+ throw (IOException) t;
+ }
+ }
+
protected ContainerManagementProtocol getContainerManagementProtocolProxy(
final YarnRPC rpc, org.apache.hadoop.yarn.api.records.Token nmToken,
NodeId nodeId, String user) {
@@ -351,7 +484,9 @@ public class TestContainerManagerSecurit
UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
final InetSocketAddress addr =
NetUtils.createSocketAddr(nodeId.getHost(), nodeId.getPort());
- ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
+ if (nmToken != null) {
+ ugi.addToken(ConverterUtils.convertFromYarn(nmToken, addr));
+ }
proxy = ugi
.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Mon Aug 12 21:25:49 2013
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<name>hadoop-yarn-server-web-proxy</name>
<properties>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml Mon Aug 12 21:25:49 2013
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<name>hadoop-yarn-server</name>
<packaging>pom</packaging>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml Mon Aug 12 21:25:49 2013
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-site</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<name>hadoop-yarn-site</name>
<properties>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Mon Aug 12 21:25:49 2013
@@ -1119,6 +1119,7 @@ ResourceManager REST API's.
* startedTimeEnd - applications with start time ending with this time, specified in ms since epoch
* finishedTimeBegin - applications with finish time beginning with this time, specified in ms since epoch
* finishedTimeEnd - applications with finish time ending with this time, specified in ms since epoch
+ * applicationTypes - applications matching the given application types, specified as a comma-separated list.
------
** Elements of the <apps> (Applications) object
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/pom.xml Mon Aug 12 21:25:49 2013
@@ -16,12 +16,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-yarn</name>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml?rev=1513258&r1=1513257&r2=1513258&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/pom.xml Mon Aug 12 21:25:49 2013
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-project</artifactId>
- <version>2.2.0-SNAPSHOT</version>
+ <version>2.3.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-yarn-project</name>
<url>http://hadoop.apache.org/yarn/</url>