You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2014/10/02 19:15:15 UTC
git commit: YARN-2254. TestRMWebServicesAppsModification should run
against both CS and FS. (Zhihai Xu via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 3ef1cf187 -> 5e0b49da9
YARN-2254. TestRMWebServicesAppsModification should run against both CS and FS. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e0b49da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e0b49da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e0b49da
Branch: refs/heads/trunk
Commit: 5e0b49da9caa53814581508e589f3704592cf335
Parents: 3ef1cf1
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Oct 1 15:47:40 2014 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Oct 2 10:15:04 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../TestRMWebServicesAppsModification.java | 121 ++++++++++++++++---
2 files changed, 105 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e0b49da/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 047fb55..a67ff27 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -45,6 +45,9 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
+ YARN-2254. TestRMWebServicesAppsModification should run against both
+ CS and FS. (Zhihai Xu via kasha)
+
Release 2.6.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e0b49da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 875c83c..1812bab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -67,6 +67,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
@@ -117,6 +119,11 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
private boolean setAuthFilter = false;
+ private static final String TEST_DIR = new File(System.getProperty(
+ "test.build.data", "/tmp")).getAbsolutePath();
+ private static final String FS_ALLOC_FILE = new File(TEST_DIR,
+ "test-fs-queues.xml").getAbsolutePath();
+
public static class GuiceServletConfig extends GuiceServletContextListener {
@Override
@@ -153,11 +160,14 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
- private class TestServletModule extends ServletModule {
+ private abstract class TestServletModule extends ServletModule {
public Configuration conf = new Configuration();
+ public abstract void configureScheduler();
+
@Override
protected void configureServlets() {
+ configureScheduler();
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
@@ -176,8 +186,63 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
}
}
- private Injector getNoAuthInjector() {
- return Guice.createInjector(new TestServletModule() {
+ private class CapTestServletModule extends TestServletModule {
+ @Override
+ public void configureScheduler() {
+ conf.set("yarn.resourcemanager.scheduler.class",
+ CapacityScheduler.class.getName());
+ }
+ }
+
+ private class FairTestServletModule extends TestServletModule {
+ @Override
+ public void configureScheduler() {
+ try {
+ PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"root\">");
+ out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
+ out.println(" <queue name=\"default\">");
+ out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+ } catch(IOException e) {
+ }
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
+ conf.set("yarn.resourcemanager.scheduler.class",
+ FairScheduler.class.getName());
+ }
+ }
+
+ private Injector getNoAuthInjectorCap() {
+ return Guice.createInjector(new CapTestServletModule() {
+ @Override
+ protected void configureServlets() {
+ setAuthFilter = false;
+ super.configureServlets();
+ }
+ });
+ }
+
+ private Injector getSimpleAuthInjectorCap() {
+ return Guice.createInjector(new CapTestServletModule() {
+ @Override
+ protected void configureServlets() {
+ setAuthFilter = true;
+ conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+ // set the admin acls otherwise all users are considered admins
+ // and we can't test authorization
+ conf.setStrings(YarnConfiguration.YARN_ADMIN_ACL, "testuser1");
+ super.configureServlets();
+ }
+ });
+ }
+
+ private Injector getNoAuthInjectorFair() {
+ return Guice.createInjector(new FairTestServletModule() {
@Override
protected void configureServlets() {
setAuthFilter = false;
@@ -186,8 +251,8 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
});
}
- private Injector getSimpleAuthInjector() {
- return Guice.createInjector(new TestServletModule() {
+ private Injector getSimpleAuthInjectorFair() {
+ return Guice.createInjector(new FairTestServletModule() {
@Override
protected void configureServlets() {
setAuthFilter = true;
@@ -202,7 +267,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
@Parameters
public static Collection<Object[]> guiceConfigs() {
- return Arrays.asList(new Object[][] { { 0 }, { 1 } });
+ return Arrays.asList(new Object[][] { { 0 }, { 1 }, { 2 }, { 3 } });
}
@Before
@@ -221,10 +286,20 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
switch (run) {
case 0:
default:
- injector = getNoAuthInjector();
+ // No Auth Capacity Scheduler
+ injector = getNoAuthInjectorCap();
break;
case 1:
- injector = getSimpleAuthInjector();
+ // Simple Auth Capacity Scheduler
+ injector = getSimpleAuthInjectorCap();
+ break;
+ case 2:
+ // No Auth Fair Scheduler
+ injector = getNoAuthInjectorFair();
+ break;
+ case 3:
+ // Simple Auth Fair Scheduler
+ injector = getSimpleAuthInjectorFair();
break;
}
}
@@ -273,7 +348,7 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
rm.stop();
}
- @Test(timeout = 90000)
+ @Test(timeout = 120000)
public void testSingleAppKill() throws Exception {
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
@@ -445,20 +520,25 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
assertTrue(msg, valid);
}
- @Test(timeout = 30000)
+ @Test(timeout = 60000)
public void testSingleAppKillUnauthorized() throws Exception {
boolean isCapacityScheduler =
rm.getResourceScheduler() instanceof CapacityScheduler;
- assumeTrue("Currently this test is only supported on CapacityScheduler",
- isCapacityScheduler);
-
- // default root queue allows anyone to have admin acl
- CapacitySchedulerConfiguration csconf =
- new CapacitySchedulerConfiguration();
- csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
- csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
- rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
+ boolean isFairScheduler =
+ rm.getResourceScheduler() instanceof FairScheduler;
+ assumeTrue("This test is only supported on Capacity and Fair Scheduler",
+ isCapacityScheduler || isFairScheduler);
+ // FairScheduler use ALLOCATION_FILE to configure ACL
+ if (isCapacityScheduler) {
+ // default root queue allows anyone to have admin acl
+ CapacitySchedulerConfiguration csconf =
+ new CapacitySchedulerConfiguration();
+ csconf.setAcl("root", QueueACL.ADMINISTER_QUEUE, "someuser");
+ csconf.setAcl("root.default", QueueACL.ADMINISTER_QUEUE, "someuser");
+ rm.getResourceScheduler().reinitialize(csconf, rm.getRMContext());
+ }
+
rm.start();
MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
@@ -733,6 +813,9 @@ public class TestRMWebServicesAppsModification extends JerseyTest {
assertEquals(appName, app.getName());
assertEquals(webserviceUserName, app.getUser());
assertEquals(2, app.getMaxAppAttempts());
+ if (app.getQueue().contains("root.")) {
+ queueName = "root." + queueName;
+ }
assertEquals(queueName, app.getQueue());
assertEquals(appType, app.getApplicationType());
assertEquals(tags, app.getApplicationTags());